rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-26 18:00:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(vector): add conversion between vector and string (#5029)

Browse Source 
* feat(vector): add conversion between vector and string

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* fix sqlness

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

* address comments

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>

---------

Signed-off-by: Zhenchi <zhongzc_arch@outlook.com>
This commit is contained in:
Zhenchi
2024-11-20 16:42:00 +08:00
committed by GitHub
parent 027284ed1b
commit 0aab68c23b
17 changed files with 655 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/datatypes/src/types/vector_type.rs
View File

@@ -102,7 +102,7 @@ pub fn vector_type_value_to_string(val: &[u8], dim: u32) -> Result<String> {
/// Parses a string to a vector type value
/// Valid input format: "[1.0,2.0,3.0]", "[1.0, 2.0, 3.0]"
pub fn parse_string_to_vector_type_value(s: &str, dim: u32) -> Result<Vec<u8>> {
pub fn parse_string_to_vector_type_value(s: &str, dim: Option<u32>) -> Result<Vec<u8>> {
// Trim the brackets
let trimmed = s.trim();
if !trimmed.starts_with('[') || !trimmed.ends_with(']') {
@@ -115,7 +115,7 @@ pub fn parse_string_to_vector_type_value(s: &str, dim: u32) -> Result<Vec<u8>> {
let content = trimmed[1..trimmed.len() - 1].trim();
if content.is_empty() {
if dim != 0 {
if dim.map_or(false, |d| d != 0) {
return InvalidVectorSnafu {
msg: format!("Failed to parse {s} to Vector value: wrong dimension"),
}
@@ -139,7 +139,7 @@ pub fn parse_string_to_vector_type_value(s: &str, dim: u32) -> Result<Vec<u8>> {
.collect::<Result<Vec<f32>>>()?;
// Check dimension
if elements.len() != dim as usize {
if dim.map_or(false, |d| d as usize != elements.len()) {
return InvalidVectorSnafu {
msg: format!("Failed to parse {s} to Vector value: wrong dimension"),
}
@@ -180,7 +180,7 @@ mod tests {
];
for (s, expected) in cases.iter() {
let val = parse_string_to_vector_type_value(s, dim).unwrap();
let val = parse_string_to_vector_type_value(s, Some(dim)).unwrap();
let s = vector_type_value_to_string(&val, dim).unwrap();
assert_eq!(s, *expected);
}
@@ -188,7 +188,7 @@ mod tests {
let dim = 0;
let cases = [("[]", "[]"), ("[ ]", "[]"), ("[ ]", "[]")];
for (s, expected) in cases.iter() {
let val = parse_string_to_vector_type_value(s, dim).unwrap();
let val = parse_string_to_vector_type_value(s, Some(dim)).unwrap();
let s = vector_type_value_to_string(&val, dim).unwrap();
assert_eq!(s, *expected);
}
@@ -211,15 +211,15 @@ mod tests {
fn test_parse_string_to_vector_type_value_not_properly_enclosed_in_brackets() {
let dim = 3;
let s = "1.0,2.0,3.0";
let res = parse_string_to_vector_type_value(s, dim);
let res = parse_string_to_vector_type_value(s, Some(dim));
assert!(res.is_err());
let s = "[1.0,2.0,3.0";
let res = parse_string_to_vector_type_value(s, dim);
let res = parse_string_to_vector_type_value(s, Some(dim));
assert!(res.is_err());
let s = "1.0,2.0,3.0]";
let res = parse_string_to_vector_type_value(s, dim);
let res = parse_string_to_vector_type_value(s, Some(dim));
assert!(res.is_err());
}
@@ -227,7 +227,7 @@ mod tests {
fn test_parse_string_to_vector_type_value_wrong_dimension() {
let dim = 3;
let s = "[1.0,2.0]";
let res = parse_string_to_vector_type_value(s, dim);
let res = parse_string_to_vector_type_value(s, Some(dim));
assert!(res.is_err());
}
@@ -235,7 +235,7 @@ mod tests {
fn test_parse_string_to_vector_type_value_elements_are_not_all_float32() {
let dim = 3;
let s = "[1.0,2.0,ah]";
let res = parse_string_to_vector_type_value(s, dim);
let res = parse_string_to_vector_type_value(s, Some(dim));
assert!(res.is_err());
}
}

2
src/datatypes/src/vectors/binary.rs
View File

@@ -80,7 +80,7 @@ impl BinaryVector {
let v = if let Some(binary) = binary {
let bytes_size = dim as usize * std::mem::size_of::<f32>();
if let Ok(s) = String::from_utf8(binary.to_vec()) {
let v = parse_string_to_vector_type_value(&s, dim)?;
let v = parse_string_to_vector_type_value(&s, Some(dim))?;
Some(v)
} else if binary.len() == dim as usize * std::mem::size_of::<f32>() {
Some(binary.to_vec())