feat: struct value and vector (#7033)

* feat: struct value

Signed-off-by: Ning Sun <sunning@greptime.com>

* feat: update for proto module

* feat: wip struct type

* feat: implement more vector operations

* feat: make datatype and api

* feat: reoslve some compilation issues

* feat: resolve all compilation issues

* chore: format update

* test: resolve tests

* test: test and refactor value-to-pb

* feat: add more tests and fix for value types

* chore: remove dbg

* feat: test and fix iterator

* fix: resolve struct_type issue

* refactor: use vec for struct items

* chore: update proto to main branch

* refactor: address some of review issues

* refactor: update for further review

* Add validation on new methods

* feat: update struct/list json serialization

* refactor: reimplement get in struct_vector

* refactor: struct vector functions

* refactor: fix lint issue

* refactor: address review comments

---------

Signed-off-by: Ning Sun <sunning@greptime.com>

src/common/function/src/helper.rs

@@ -38,7 +38,7 @@ pub(crate) fn one_of_sigs2(args1: Vec<DataType>, args2: Vec<DataType>) -> Signat
 
 /// Cast a [`ValueRef`] to u64, returns `None` if fails
 pub fn cast_u64(value: &ValueRef) -> Result<Option<u64>> {
-    cast((*value).into(), &ConcreteDataType::uint64_datatype())
+    cast(value.clone().into(), &ConcreteDataType::uint64_datatype())
         .context(InvalidInputTypeSnafu {
             err_msg: format!(
                 "Failed to cast input into uint64, actual type: {:#?}",
@@ -50,7 +50,7 @@ pub fn cast_u64(value: &ValueRef) -> Result<Option<u64>> {
 
 /// Cast a [`ValueRef`] to u32, returns `None` if fails
 pub fn cast_u32(value: &ValueRef) -> Result<Option<u32>> {
-    cast((*value).into(), &ConcreteDataType::uint32_datatype())
+    cast(value.clone().into(), &ConcreteDataType::uint32_datatype())
         .context(InvalidInputTypeSnafu {
             err_msg: format!(
                 "Failed to cast input into uint32, actual type: {:#?}",

src/common/function/src/scalars/expression/binary.rs

@@ -41,7 +41,7 @@ where
             let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(right.inner()) };
             let b = right.get_data(0);
 
-            let it = left.iter_data().map(|a| f(a, b, ctx));
+            let it = left.iter_data().map(|a| f(a, b.clone(), ctx));
             <O as Scalar>::VectorType::from_owned_iterator(it)
         }
 
@@ -62,7 +62,7 @@ where
             let a = left.get_data(0);
 
             let right: &<R as Scalar>::VectorType = unsafe { Helper::static_cast(r) };
-            let it = right.iter_data().map(|b| f(a, b, ctx));
+            let it = right.iter_data().map(|b| f(a.clone(), b, ctx));
             <O as Scalar>::VectorType::from_owned_iterator(it)
         }
 

src/common/grpc-expr/src/insert.rs

@@ -46,7 +46,7 @@ pub(crate) fn add_values_to_builder(
                 Some(true) => builder.push_null(),
                 _ => {
                     builder
-                        .try_push_value_ref(values[idx_of_values].as_value_ref())
+                        .try_push_value_ref(&values[idx_of_values].as_value_ref())
                         .context(CreateVectorSnafu)?;
                     idx_of_values += 1
                 }

src/common/grpc-expr/src/util.rs

@@ -167,7 +167,7 @@ pub fn build_create_table_expr(
             default_constraint: vec![],
             semantic_type,
             comment: String::new(),
-            datatype_extension: *datatype_extension,
+            datatype_extension: datatype_extension.clone(),
             options: options.clone(),
         });
     }
@@ -209,7 +209,7 @@ pub fn extract_new_columns(
                 default_constraint: vec![],
                 semantic_type: expr.semantic_type,
                 comment: String::new(),
-                datatype_extension: *expr.datatype_extension,
+                datatype_extension: expr.datatype_extension.clone(),
                 options: expr.options.clone(),
             });
             AddColumn {
@@ -425,7 +425,7 @@ mod tests {
             ConcreteDataType::from(
                 ColumnDataTypeWrapper::try_new(
                     decimal_column.data_type,
-                    decimal_column.datatype_extension,
+                    decimal_column.datatype_extension.clone(),
                 )
                 .unwrap()
             )
@@ -520,6 +520,7 @@ mod tests {
                         .as_ref()
                         .unwrap()
                         .datatype_extension
+                        .clone()
                 )
                 .unwrap()
             )

src/common/macro/src/admin_fn.rs

@@ -325,7 +325,7 @@ fn build_struct(
                     let result = #fn_name(handler, query_ctx, &[]).await
                         .map_err(|e| datafusion_common::DataFusionError::Execution(format!("Function execution error: {}", e.output_msg())))?;
 
-                    builder.push_value_ref(result.as_value_ref());
+                    builder.push_value_ref(&result.as_value_ref());
                 } else {
                     for i in 0..rows_num {
                         let args: Vec<_> = columns.iter()
@@ -335,7 +335,7 @@ fn build_struct(
                         let result = #fn_name(handler, query_ctx, &args).await
                             .map_err(|e| datafusion_common::DataFusionError::Execution(format!("Function execution error: {}", e.output_msg())))?;
 
-                        builder.push_value_ref(result.as_value_ref());
+                        builder.push_value_ref(&result.as_value_ref());
                     }
                 }
 

src/common/macro/src/row/schema.rs

@@ -90,6 +90,24 @@ fn impl_schema_method(fields: &[ParsedField<'_>]) -> Result<TokenStream2> {
                                 Some(ColumnDataTypeExtension { type_ext: Some(TypeExt::VectorType(VectorTypeExtension { dim: #dim })) })
                             }
                         }
+                        Some(TypeExt::ListType(ext)) => {
+                            let item_type = syn::Ident::new(&ext.datatype.to_string(), ident.span());
+                            quote! {
+                                Some(ColumnDataTypeExtension { type_ext: Some(TypeExt::ListType(ListTypeExtension { item_type: #item_type })) })
+                            }
+                        }
+                        Some(TypeExt::StructType(ext)) => {
+                            let fields = ext.fields.iter().map(|field| {
+                                let field_name = syn::Ident::new(&field.name.to_string(), ident.span());
+                                let field_type = syn::Ident::new(&field.datatype.to_string(), ident.span());
+                                quote! {
+                                    StructField { name: #field_name, type_: #field_type }
+                                }
+                            }).collect::<Vec<_>>();
+                            quote! {
+                                Some(ColumnDataTypeExtension { type_ext: Some(TypeExt::StructType(StructTypeExtension { fields: [#(#fields),*] })) })
+                            }
+                        }
                         None => {
                             quote! { None }
                         }

src/common/macro/src/row/utils.rs

@@ -184,7 +184,7 @@ pub(crate) fn convert_semantic_type_to_proto_semantic_type(
     }
 }
 
-#[derive(Debug, Clone, Copy)]
+#[derive(Debug, Clone)]
 pub(crate) struct ColumnDataTypeWithExtension {
     pub(crate) data_type: ColumnDataType,
     pub(crate) extension: Option<ColumnDataTypeExtension>,
@@ -260,7 +260,10 @@ pub(crate) fn get_column_data_type(
     infer_column_data_type: &Option<ColumnDataTypeWithExtension>,
     attribute: &ColumnAttribute,
 ) -> Option<ColumnDataTypeWithExtension> {
-    attribute.datatype.or(*infer_column_data_type)
+    attribute
+        .datatype
+        .clone()
+        .or_else(|| infer_column_data_type.clone())
 }
 
 /// Convert a column data type to a value data ident.
@@ -304,5 +307,7 @@ pub(crate) fn convert_column_data_type_to_value_data_ident(
         // Json is a special case, it is actually a string column.
         ColumnDataType::Json => format_ident!("StringValue"),
         ColumnDataType::Vector => format_ident!("VectorValue"),
+        ColumnDataType::List => format_ident!("ListValue"),
+        ColumnDataType::Struct => format_ident!("StructValue"),
     }
 }

src/common/meta/src/ddl/create_table_template.rs

@@ -90,7 +90,7 @@ pub(crate) fn build_template(create_table_expr: &CreateTableExpr) -> Result<Crea
                     default_constraint: c.default_constraint.clone(),
                     semantic_type: semantic_type as i32,
                     comment: String::new(),
-                    datatype_extension: c.datatype_extension,
+                    datatype_extension: c.datatype_extension.clone(),
                     options: c.options.clone(),
                 }),
                 column_id: i as u32,

src/common/sql/src/convert.rs

@@ -220,7 +220,7 @@ pub fn sql_value_to_value(
                 _ => return InvalidUnaryOpSnafu { unary_op, value }.fail(),
             },
 
-            Value::String(_) | Value::Binary(_) | Value::List(_) => {
+            Value::String(_) | Value::Binary(_) | Value::List(_) | Value::Struct(_) => {
                 return InvalidUnaryOpSnafu { unary_op, value }.fail();
             }
         }