feat: introduce vector type (#4964)

* feat: introduce vector type

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* test: fix prepared stmt

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* test: add grpc test

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* test: parse vector value

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* test: column to row

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* test: sqlness

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* fix: merge issue

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* refactor: add check for bytes size

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* Update tests/cases/standalone/common/types/vector/vector.sql

Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

* chore: update proto

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* chore: simplify cargo

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* chore: address comment

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

---------

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
Co-authored-by: Ruihang Xia <waynestxia@gmail.com>

src/datatypes/src/data_type.rs

@@ -36,7 +36,7 @@ use crate::types::{
     IntervalDayTimeType, IntervalMonthDayNanoType, IntervalType, IntervalYearMonthType, JsonType,
     ListType, NullType, StringType, TimeMillisecondType, TimeType, TimestampMicrosecondType,
     TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, TimestampType,
-    UInt16Type, UInt32Type, UInt64Type, UInt8Type,
+    UInt16Type, UInt32Type, UInt64Type, UInt8Type, VectorType,
 };
 use crate::value::Value;
 use crate::vectors::MutableVector;
@@ -84,6 +84,9 @@ pub enum ConcreteDataType {
 
     // JSON type:
     Json(JsonType),
+
+    // Vector type:
+    Vector(VectorType),
 }
 
 impl fmt::Display for ConcreteDataType {
@@ -132,6 +135,7 @@ impl fmt::Display for ConcreteDataType {
             ConcreteDataType::List(v) => write!(f, "{}", v.name()),
             ConcreteDataType::Dictionary(v) => write!(f, "{}", v.name()),
             ConcreteDataType::Json(v) => write!(f, "{}", v.name()),
+            ConcreteDataType::Vector(v) => write!(f, "{}", v.name()),
         }
     }
 }
@@ -167,6 +171,7 @@ impl ConcreteDataType {
                 | ConcreteDataType::Decimal128(_)
                 | ConcreteDataType::Binary(_)
                 | ConcreteDataType::Json(_)
+                | ConcreteDataType::Vector(_)
         )
     }
 
@@ -225,6 +230,10 @@ impl ConcreteDataType {
         matches!(self, ConcreteDataType::Json(_))
     }
 
+    pub fn is_vector(&self) -> bool {
+        matches!(self, ConcreteDataType::Vector(_))
+    }
+
     pub fn numerics() -> Vec<ConcreteDataType> {
         vec![
             ConcreteDataType::int8_datatype(),
@@ -334,6 +343,13 @@ impl ConcreteDataType {
         }
     }
 
+    pub fn as_vector(&self) -> Option<VectorType> {
+        match self {
+            ConcreteDataType::Vector(v) => Some(*v),
+            _ => None,
+        }
+    }
+
     /// Checks if the data type can cast to another data type.
     pub fn can_arrow_type_cast_to(&self, to_type: &ConcreteDataType) -> bool {
         let array = arrow_array::new_empty_array(&self.as_arrow_type());
@@ -564,6 +580,14 @@ impl ConcreteDataType {
     pub fn decimal128_default_datatype() -> ConcreteDataType {
         Self::decimal128_datatype(DECIMAL128_MAX_PRECISION, DECIMAL_DEFAULT_SCALE)
     }
+
+    pub fn vector_datatype(dim: u32) -> ConcreteDataType {
+        ConcreteDataType::Vector(VectorType::new(dim))
+    }
+
+    pub fn vector_default_datatype() -> ConcreteDataType {
+        Self::vector_datatype(0)
+    }
 }
 
 /// Data type abstraction.
@@ -757,6 +781,7 @@ mod tests {
         assert!(ConcreteDataType::duration_microsecond_datatype().is_stringifiable());
         assert!(ConcreteDataType::duration_nanosecond_datatype().is_stringifiable());
         assert!(ConcreteDataType::decimal128_datatype(10, 2).is_stringifiable());
+        assert!(ConcreteDataType::vector_default_datatype().is_stringifiable());
     }
 
     #[test]
@@ -909,5 +934,9 @@ mod tests {
             .to_string(),
             "Dictionary<Int32, String>"
         );
+        assert_eq!(
+            ConcreteDataType::vector_datatype(3).to_string(),
+            "Vector(3)"
+        );
     }
 }

src/datatypes/src/error.rs

@@ -196,6 +196,13 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Invalid Vector: {}", msg))]
+    InvalidVector {
+        msg: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
+
     #[snafu(display("Value exceeds the precision {} bound", precision))]
     ValueExceedsPrecision {
         precision: u8,
@@ -213,6 +220,12 @@ pub enum Error {
         location: Location,
     },
 
+    #[snafu(display("Failed to parse extended type in metadata: {}", value))]
+    ParseExtendedType {
+        value: String,
+        #[snafu(implicit)]
+        location: Location,
+    },
     #[snafu(display("Invalid fulltext option: {}", msg))]
     InvalidFulltextOption {
         msg: String,
@@ -238,6 +251,7 @@ impl ErrorExt for Error {
             | InvalidTimestampPrecision { .. }
             | InvalidPrecisionOrScale { .. }
             | InvalidJson { .. }
+            | InvalidVector { .. }
             | InvalidFulltextOption { .. } => StatusCode::InvalidArguments,
 
             ValueExceedsPrecision { .. }
@@ -253,7 +267,8 @@ impl ErrorExt for Error {
             | ProjectArrowSchema { .. }
             | ToScalarValue { .. }
             | TryFromValue { .. }
-            | ConvertArrowArrayToScalars { .. } => StatusCode::Internal,
+            | ConvertArrowArrayToScalars { .. }
+            | ParseExtendedType { .. } => StatusCode::Internal,
         }
     }
 

src/datatypes/src/schema.rs

@@ -21,11 +21,12 @@ use std::fmt;
 use std::sync::Arc;
 
 use arrow::datatypes::{Field, Schema as ArrowSchema};
+use column_schema::ColumnExtType;
 use datafusion_common::DFSchemaRef;
 use snafu::{ensure, ResultExt};
 
 use crate::error::{self, DuplicateColumnSnafu, Error, ProjectArrowSchemaSnafu, Result};
-use crate::prelude::DataType;
+use crate::prelude::ConcreteDataType;
 pub use crate::schema::column_schema::{
     ColumnSchema, FulltextAnalyzer, FulltextOptions, Metadata,
     COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE, COLUMN_FULLTEXT_OPT_KEY_ANALYZER,
@@ -263,9 +264,14 @@ fn collect_fields(column_schemas: &[ColumnSchema]) -> Result<FieldsAndIndices> {
         }
         let mut field = Field::try_from(column_schema)?;
 
-        // Json column performs the same as binary column in Arrow, so we need to mark it
-        if column_schema.data_type.is_json() {
-            let metadata = HashMap::from([(TYPE_KEY.to_string(), column_schema.data_type.name())]);
+        // Column with type Json or Vector performs the same as binary column in Arrow, so we need to mark it
+        let extype = match column_schema.data_type {
+            ConcreteDataType::Json(_) => Some(ColumnExtType::Json),
+            ConcreteDataType::Vector(d) => Some(ColumnExtType::Vector(d.dim)),
+            _ => None,
+        };
+        if let Some(extype) = extype {
+            let metadata = HashMap::from([(TYPE_KEY.to_string(), extype.to_string())]);
             field = field.with_metadata(metadata);
         }
         fields.push(field);

src/datatypes/src/schema/column_schema.rs

@@ -14,6 +14,7 @@
 
 use std::collections::HashMap;
 use std::fmt;
+use std::str::FromStr;
 
 use arrow::datatypes::Field;
 use serde::{Deserialize, Serialize};
@@ -21,10 +22,9 @@ use snafu::{ensure, ResultExt};
 use sqlparser_derive::{Visit, VisitMut};
 
 use crate::data_type::{ConcreteDataType, DataType};
-use crate::error::{self, Error, InvalidFulltextOptionSnafu, Result};
+use crate::error::{self, Error, InvalidFulltextOptionSnafu, ParseExtendedTypeSnafu, Result};
 use crate::schema::constraint::ColumnDefaultConstraint;
 use crate::schema::TYPE_KEY;
-use crate::types::JSON_TYPE_NAME;
 use crate::value::Value;
 use crate::vectors::VectorRef;
 
@@ -300,17 +300,57 @@ impl ColumnSchema {
     }
 }
 
+/// Column extended type set in column schema's metadata.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ColumnExtType {
+    /// Json type.
+    Json,
+
+    /// Vector type with dimension.
+    Vector(u32),
+}
+
+impl fmt::Display for ColumnExtType {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match self {
+            ColumnExtType::Json => write!(f, "Json"),
+            ColumnExtType::Vector(dim) => write!(f, "Vector({})", dim),
+        }
+    }
+}
+
+impl FromStr for ColumnExtType {
+    type Err = String;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        match s {
+            "Json" => Ok(ColumnExtType::Json),
+            _ if s.starts_with("Vector(") && s.ends_with(')') => s[7..s.len() - 1]
+                .parse::<u32>()
+                .map(ColumnExtType::Vector)
+                .map_err(|_| "Invalid dimension for Vector".to_string()),
+            _ => Err("Unknown variant".to_string()),
+        }
+    }
+}
+
 impl TryFrom<&Field> for ColumnSchema {
     type Error = Error;
 
     fn try_from(field: &Field) -> Result<ColumnSchema> {
         let mut data_type = ConcreteDataType::try_from(field.data_type())?;
         // Override the data type if it is specified in the metadata.
-        if field.metadata().contains_key(TYPE_KEY) {
-            data_type = match field.metadata().get(TYPE_KEY).unwrap().as_str() {
-                JSON_TYPE_NAME => ConcreteDataType::json_datatype(),
-                _ => data_type,
-            };
+        if let Some(s) = field.metadata().get(TYPE_KEY) {
+            let extype = ColumnExtType::from_str(s)
+                .map_err(|_| ParseExtendedTypeSnafu { value: s }.build())?;
+            match extype {
+                ColumnExtType::Json => {
+                    data_type = ConcreteDataType::json_datatype();
+                }
+                ColumnExtType::Vector(dim) => {
+                    data_type = ConcreteDataType::vector_datatype(dim);
+                }
+            }
         }
         let mut metadata = field.metadata().clone();
         let default_constraint = match metadata.remove(DEFAULT_CONSTRAINT_KEY) {
@@ -661,5 +701,24 @@ mod tests {
             column_schema.metadata.get(TYPE_KEY).unwrap(),
             &ConcreteDataType::json_datatype().name()
         );
+
+        let field = Field::new("test", ArrowDataType::Binary, true);
+        let field = field.with_metadata(Metadata::from([(
+            TYPE_KEY.to_string(),
+            ConcreteDataType::vector_datatype(3).name(),
+        )]));
+        let column_schema = ColumnSchema::try_from(&field).unwrap();
+        assert_eq!("test", column_schema.name);
+        assert_eq!(
+            ConcreteDataType::vector_datatype(3),
+            column_schema.data_type
+        );
+        assert!(column_schema.is_nullable);
+        assert!(!column_schema.is_time_index);
+        assert!(column_schema.default_constraint.is_none());
+        assert_eq!(
+            column_schema.metadata.get(TYPE_KEY).unwrap(),
+            &ConcreteDataType::vector_datatype(3).name()
+        );
     }
 }

src/datatypes/src/schema/constraint.rs

@@ -94,7 +94,7 @@ impl ColumnDefaultConstraint {
                     // Whether the value could be nullable has been checked before, only need
                     // to check the type compatibility here.
                     ensure!(
-                        data_type.logical_type_id() == v.logical_type_id(),
+                        value_type_match(data_type, v.data_type()),
                         error::DefaultValueTypeSnafu {
                             reason: format!(
                                 "column has type {:?} but default value has type {:?}",
@@ -215,6 +215,17 @@ fn create_current_timestamp_vector(
     }
 }
 
+fn value_type_match(column_type: &ConcreteDataType, value_type: ConcreteDataType) -> bool {
+    match (column_type, value_type) {
+        (ct, vt) if ct.logical_type_id() == vt.logical_type_id() => true,
+        // Vector and Json type is encoded as binary
+        (ConcreteDataType::Vector(_) | ConcreteDataType::Json(_), ConcreteDataType::Binary(_)) => {
+            true
+        }
+        _ => false,
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;

src/datatypes/src/type_id.rs

@@ -70,6 +70,8 @@ pub enum LogicalTypeId {
     Dictionary,
 
     Json,
+
+    Vector,
 }
 
 impl LogicalTypeId {
@@ -129,6 +131,7 @@ impl LogicalTypeId {
             LogicalTypeId::DurationNanosecond => ConcreteDataType::duration_nanosecond_datatype(),
             LogicalTypeId::Decimal128 => ConcreteDataType::decimal128_default_datatype(),
             LogicalTypeId::Json => ConcreteDataType::json_datatype(),
+            LogicalTypeId::Vector => ConcreteDataType::vector_default_datatype(),
         }
     }
 }

src/datatypes/src/types.rs

@@ -28,6 +28,7 @@ mod primitive_type;
 mod string_type;
 mod time_type;
 mod timestamp_type;
+mod vector_type;
 
 pub use binary_type::BinaryType;
 pub use boolean_type::BooleanType;
@@ -58,3 +59,4 @@ pub use timestamp_type::{
     TimestampMicrosecondType, TimestampMillisecondType, TimestampNanosecondType,
     TimestampSecondType, TimestampType,
 };
+pub use vector_type::{parse_string_to_vector_type_value, vector_type_value_to_string, VectorType};

src/datatypes/src/types/vector_type.rs

@@ -0,0 +1,237 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use arrow::datatypes::DataType as ArrowDataType;
+use common_base::bytes::Bytes;
+use serde::{Deserialize, Serialize};
+
+use crate::data_type::DataType;
+use crate::error::{InvalidVectorSnafu, Result};
+use crate::scalars::ScalarVectorBuilder;
+use crate::type_id::LogicalTypeId;
+use crate::value::Value;
+use crate::vectors::{BinaryVectorBuilder, MutableVector};
+
+/// `VectorType` is a data type for vector data with a fixed dimension.
+/// The type of items in the vector is float32.
+/// It is stored as binary data that contains the concatenated float32 values.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
+pub struct VectorType {
+    pub dim: u32,
+}
+
+impl VectorType {
+    pub fn new(dim: u32) -> Self {
+        Self { dim }
+    }
+}
+
+impl DataType for VectorType {
+    fn name(&self) -> String {
+        format!("Vector({})", self.dim)
+    }
+
+    fn logical_type_id(&self) -> LogicalTypeId {
+        LogicalTypeId::Vector
+    }
+
+    fn default_value(&self) -> Value {
+        Bytes::default().into()
+    }
+
+    fn as_arrow_type(&self) -> ArrowDataType {
+        ArrowDataType::Binary
+    }
+
+    fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector> {
+        Box::new(BinaryVectorBuilder::with_capacity(capacity))
+    }
+
+    fn try_cast(&self, from: Value) -> Option<Value> {
+        match from {
+            Value::Binary(v) => Some(Value::Binary(v)),
+            _ => None,
+        }
+    }
+}
+
+/// Converts a vector type value to string
+/// for example: [1.0, 2.0, 3.0] -> "[1.0,2.0,3.0]"
+pub fn vector_type_value_to_string(val: &[u8], dim: u32) -> Result<String> {
+    let expected_len = dim as usize * std::mem::size_of::<f32>();
+    if val.len() != expected_len {
+        return InvalidVectorSnafu {
+            msg: format!(
+                "Failed to convert Vector value to string: wrong byte size, expected {}, got {}",
+                expected_len,
+                val.len()
+            ),
+        }
+        .fail();
+    }
+
+    if dim == 0 {
+        return Ok("[]".to_string());
+    }
+
+    let elements = unsafe {
+        std::slice::from_raw_parts(
+            val.as_ptr() as *const f32,
+            val.len() / std::mem::size_of::<f32>(),
+        )
+    };
+
+    let mut s = String::from("[");
+    for (i, e) in elements.iter().enumerate() {
+        if i > 0 {
+            s.push(',');
+        }
+        s.push_str(&e.to_string());
+    }
+    s.push(']');
+    Ok(s)
+}
+
+/// Parses a string to a vector type value
+/// Valid input format: "[1.0,2.0,3.0]", "[1.0, 2.0, 3.0]"
+pub fn parse_string_to_vector_type_value(s: &str, dim: u32) -> Result<Vec<u8>> {
+    // Trim the brackets
+    let trimmed = s.trim();
+    if !trimmed.starts_with('[') || !trimmed.ends_with(']') {
+        return InvalidVectorSnafu {
+            msg: format!("Failed to parse {s} to Vector value: not properly enclosed in brackets"),
+        }
+        .fail();
+    }
+    // Remove the brackets
+    let content = trimmed[1..trimmed.len() - 1].trim();
+
+    if content.is_empty() {
+        if dim != 0 {
+            return InvalidVectorSnafu {
+                msg: format!("Failed to parse {s} to Vector value: wrong dimension"),
+            }
+            .fail();
+        }
+        return Ok(vec![]);
+    }
+
+    let elements = content
+        .split(',')
+        .map(|e| {
+            e.trim().parse::<f32>().map_err(|_| {
+                InvalidVectorSnafu {
+                    msg: format!(
+                        "Failed to parse {s} to Vector value: elements are not all float32"
+                    ),
+                }
+                .build()
+            })
+        })
+        .collect::<Result<Vec<f32>>>()?;
+
+    // Check dimension
+    if elements.len() != dim as usize {
+        return InvalidVectorSnafu {
+            msg: format!("Failed to parse {s} to Vector value: wrong dimension"),
+        }
+        .fail();
+    }
+
+    // Convert Vec<f32> to Vec<u8>
+    let bytes = unsafe {
+        std::slice::from_raw_parts(
+            elements.as_ptr() as *const u8,
+            elements.len() * std::mem::size_of::<f32>(),
+        )
+        .to_vec()
+    };
+
+    Ok(bytes)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_conversion_between_string_and_vector_type_value() {
+        let dim = 3;
+
+        let cases = [
+            ("[1.0,2.0,3]", "[1,2,3]"),
+            ("[0.0 , 0.0 , 0.0]", "[0,0,0]"),
+            ("[3.4028235e38, -3.4028235e38, 1.1754944e-38]", "[340282350000000000000000000000000000000,-340282350000000000000000000000000000000,0.000000000000000000000000000000000000011754944]"),
+        ];
+
+        for (s, expected) in cases.iter() {
+            let val = parse_string_to_vector_type_value(s, dim).unwrap();
+            let s = vector_type_value_to_string(&val, dim).unwrap();
+            assert_eq!(s, *expected);
+        }
+
+        let dim = 0;
+        let cases = [("[]", "[]"), ("[ ]", "[]"), ("[  ]", "[]")];
+        for (s, expected) in cases.iter() {
+            let val = parse_string_to_vector_type_value(s, dim).unwrap();
+            let s = vector_type_value_to_string(&val, dim).unwrap();
+            assert_eq!(s, *expected);
+        }
+    }
+
+    #[test]
+    fn test_vector_type_value_to_string_wrong_byte_size() {
+        let dim = 3;
+        let val = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11];
+        let res = vector_type_value_to_string(&val, dim);
+        assert!(res.is_err());
+
+        let dim = 0;
+        let val = vec![1];
+        let res = vector_type_value_to_string(&val, dim);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_parse_string_to_vector_type_value_not_properly_enclosed_in_brackets() {
+        let dim = 3;
+        let s = "1.0,2.0,3.0";
+        let res = parse_string_to_vector_type_value(s, dim);
+        assert!(res.is_err());
+
+        let s = "[1.0,2.0,3.0";
+        let res = parse_string_to_vector_type_value(s, dim);
+        assert!(res.is_err());
+
+        let s = "1.0,2.0,3.0]";
+        let res = parse_string_to_vector_type_value(s, dim);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_parse_string_to_vector_type_value_wrong_dimension() {
+        let dim = 3;
+        let s = "[1.0,2.0]";
+        let res = parse_string_to_vector_type_value(s, dim);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_parse_string_to_vector_type_value_elements_are_not_all_float32() {
+        let dim = 3;
+        let s = "[1.0,2.0,ah]";
+        let res = parse_string_to_vector_type_value(s, dim);
+        assert!(res.is_err());
+    }
+}

src/datatypes/src/value.rs

@@ -520,7 +520,9 @@ pub fn to_null_scalar_value(output_type: &ConcreteDataType) -> Result<ScalarValu
         ConcreteDataType::UInt64(_) => ScalarValue::UInt64(None),
         ConcreteDataType::Float32(_) => ScalarValue::Float32(None),
         ConcreteDataType::Float64(_) => ScalarValue::Float64(None),
-        ConcreteDataType::Binary(_) | ConcreteDataType::Json(_) => ScalarValue::Binary(None),
+        ConcreteDataType::Binary(_) | ConcreteDataType::Json(_) | ConcreteDataType::Vector(_) => {
+            ScalarValue::Binary(None)
+        }
         ConcreteDataType::String(_) => ScalarValue::Utf8(None),
         ConcreteDataType::Date(_) => ScalarValue::Date32(None),
         ConcreteDataType::DateTime(_) => ScalarValue::Date64(None),

src/datatypes/src/vectors/binary.rs

@@ -20,9 +20,10 @@ use snafu::ResultExt;
 
 use crate::arrow_array::{BinaryArray, MutableBinaryArray};
 use crate::data_type::ConcreteDataType;
-use crate::error::{self, Result};
+use crate::error::{self, InvalidVectorSnafu, Result};
 use crate::scalars::{ScalarVector, ScalarVectorBuilder};
 use crate::serialize::Serializable;
+use crate::types::parse_string_to_vector_type_value;
 use crate::value::{Value, ValueRef};
 use crate::vectors::{self, MutableVector, Validity, Vector, VectorRef};
 
@@ -66,6 +67,40 @@ impl BinaryVector {
         }
         Ok(BinaryVector::from(vector))
     }
+
+    pub fn convert_binary_to_vector(&self, dim: u32) -> Result<BinaryVector> {
+        let arrow_array = self.to_arrow_array();
+        let mut vector = vec![];
+        for binary in arrow_array
+            .as_any()
+            .downcast_ref::<BinaryArray>()
+            .unwrap()
+            .iter()
+        {
+            let v = if let Some(binary) = binary {
+                let bytes_size = dim as usize * std::mem::size_of::<f32>();
+                if let Ok(s) = String::from_utf8(binary.to_vec()) {
+                    let v = parse_string_to_vector_type_value(&s, dim)?;
+                    Some(v)
+                } else if binary.len() == dim as usize * std::mem::size_of::<f32>() {
+                    Some(binary.to_vec())
+                } else {
+                    return InvalidVectorSnafu {
+                        msg: format!(
+                            "Unexpected bytes size for vector value, expected {}, got {}",
+                            bytes_size,
+                            binary.len()
+                        ),
+                    }
+                    .fail();
+                }
+            } else {
+                None
+            };
+            vector.push(v);
+        }
+        Ok(BinaryVector::from(vector))
+    }
 }
 
 impl From<BinaryArray> for BinaryVector {
@@ -473,4 +508,45 @@ mod tests {
             .unwrap_err();
         assert_matches!(error, error::Error::InvalidJson { .. });
     }
+
+    #[test]
+    fn test_binary_vector_conversion() {
+        let dim = 3;
+        let vector = BinaryVector::from(vec![
+            Some(b"[1,2,3]".to_vec()),
+            Some(b"[4,5,6]".to_vec()),
+            Some(b"[7,8,9]".to_vec()),
+            None,
+        ]);
+        let expected = BinaryVector::from(vec![
+            Some(
+                [1.0f32, 2.0, 3.0]
+                    .iter()
+                    .flat_map(|v| v.to_le_bytes())
+                    .collect(),
+            ),
+            Some(
+                [4.0f32, 5.0, 6.0]
+                    .iter()
+                    .flat_map(|v| v.to_le_bytes())
+                    .collect(),
+            ),
+            Some(
+                [7.0f32, 8.0, 9.0]
+                    .iter()
+                    .flat_map(|v| v.to_le_bytes())
+                    .collect(),
+            ),
+            None,
+        ]);
+
+        let converted = vector.convert_binary_to_vector(dim).unwrap();
+        assert_eq!(converted.len(), expected.len());
+        for i in 0..3 {
+            assert_eq!(
+                converted.get_ref(i).as_binary().unwrap().unwrap(),
+                expected.get_ref(i).as_binary().unwrap().unwrap()
+            );
+        }
+    }
 }

src/datatypes/src/vectors/eq.rs

@@ -80,7 +80,7 @@ fn equal(lhs: &dyn Vector, rhs: &dyn Vector) -> bool {
     match lhs.data_type() {
         Null(_) => true,
         Boolean(_) => is_vector_eq!(BooleanVector, lhs, rhs),
-        Binary(_) | Json(_) => is_vector_eq!(BinaryVector, lhs, rhs),
+        Binary(_) | Json(_) | Vector(_) => is_vector_eq!(BinaryVector, lhs, rhs),
         String(_) => is_vector_eq!(StringVector, lhs, rhs),
         Date(_) => is_vector_eq!(DateVector, lhs, rhs),
         DateTime(_) => is_vector_eq!(DateTimeVector, lhs, rhs),

src/datatypes/src/vectors/operations.rs

@@ -91,10 +91,17 @@ macro_rules! impl_scalar_vector_op {
             }
 
             fn cast(&self, to_type: &ConcreteDataType) -> Result<VectorRef> {
-                if to_type == &ConcreteDataType::json_datatype() {
-                    if let Some(vector) = self.as_any().downcast_ref::<BinaryVector>() {
-                        let json_vector = vector.convert_binary_to_json()?;
-                        return Ok(Arc::new(json_vector) as VectorRef);
+                if let Some(vector) = self.as_any().downcast_ref::<BinaryVector>() {
+                    match to_type {
+                        ConcreteDataType::Json(_) => {
+                            let json_vector = vector.convert_binary_to_json()?;
+                            return Ok(Arc::new(json_vector) as VectorRef);
+                        }
+                        ConcreteDataType::Vector(d) => {
+                            let vector = vector.convert_binary_to_vector(d.dim)?;
+                            return Ok(Arc::new(vector) as VectorRef);
+                        }
+                        _ => {}
                     }
                 }
                 cast::cast_non_constant!(self, to_type)