feat: add json data type (#4619)

* feat: add json type and vector

* fix: allow to create and insert json data

* feat: udf to query json as string

* refactor: remove JsonbValue and JsonVector

* feat: show json value as strings

* chore: make ci happy

* test: adunit test and sqlness test

* refactor: use binary as grpc value of json

* fix: use non-preserve-order jsonb

* test: revert changed test

* refactor: change udf get_by_path to jq

* chore: make ci happy

* fix: distinguish binary and json in proto

* chore: delete udf for future pr

* refactor: remove Value(Json)

* chore: follow review comments

* test: some tests and checks

* test: fix unit tests

* chore: follow review comments

* chore: corresponding changes to proto

* fix: change grpc and pgsql server behavior alongside with sqlness/crud tests

* chore: follow review comments

* feat: udf of conversions between json and strings, used for grpc server

* refactor: rename to_string to json_to_string

* test: add more sqlness test for json

* chore: thanks for review :)

* Apply suggestions from code review

---------

Co-authored-by: Weny Xu <wenymedia@gmail.com>

src/sql/Cargo.toml

@@ -26,6 +26,7 @@ datatypes.workspace = true
 hex = "0.4"
 iso8601 = "0.6.1"
 itertools.workspace = true
+jsonb.workspace = true
 lazy_static.workspace = true
 regex.workspace = true
 serde_json.workspace = true

src/sql/src/statements.rs

@@ -124,6 +124,16 @@ fn parse_string_to_value(
             }
         }
         ConcreteDataType::Binary(_) => Ok(Value::Binary(s.as_bytes().into())),
+        ConcreteDataType::Json(_) => {
+            if let Ok(json) = jsonb::parse_value(s.as_bytes()) {
+                Ok(Value::Binary(json.to_vec().into()))
+            } else {
+                ParseSqlValueSnafu {
+                    msg: format!("Failed to parse {s} to Json value"),
+                }
+                .fail()
+            }
+        }
         _ => {
             unreachable!()
         }
@@ -250,7 +260,19 @@ pub fn sql_value_to_value(
         SqlValue::DoubleQuotedString(s) | SqlValue::SingleQuotedString(s) => {
             parse_string_to_value(column_name, s.clone(), data_type, timezone)?
         }
-        SqlValue::HexStringLiteral(s) => parse_hex_string(s)?,
+        SqlValue::HexStringLiteral(s) => {
+            // Should not directly write binary into json column
+            ensure!(
+                !matches!(data_type, ConcreteDataType::Json(_)),
+                ColumnTypeMismatchSnafu {
+                    column_name,
+                    expect: ConcreteDataType::binary_datatype(),
+                    actual: ConcreteDataType::json_datatype(),
+                }
+            );
+
+            parse_hex_string(s)?
+        }
         SqlValue::Placeholder(s) => return InvalidSqlValueSnafu { value: s }.fail(),
 
         // TODO(dennis): supports binary string
@@ -571,6 +593,7 @@ pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<Co
                 Ok(ConcreteDataType::decimal128_datatype(*p as u8, *s as i8))
             }
         },
+        SqlDataType::JSON => Ok(ConcreteDataType::json_datatype()),
         _ => error::SqlTypeNotSupportedSnafu {
             t: data_type.clone(),
         }
@@ -607,6 +630,7 @@ pub fn concrete_data_type_to_sql_data_type(data_type: &ConcreteDataType) -> Resu
         ConcreteDataType::Decimal128(d) => Ok(SqlDataType::Decimal(
             ExactNumberInfo::PrecisionAndScale(d.precision() as u64, d.scale() as u64),
         )),
+        ConcreteDataType::Json(_) => Ok(SqlDataType::JSON),
         ConcreteDataType::Duration(_)
         | ConcreteDataType::Null(_)
         | ConcreteDataType::List(_)
@@ -872,6 +896,35 @@ mod tests {
         );
         assert!(v.is_err());
         assert!(format!("{v:?}").contains("invalid character"), "v is {v:?}",);
+
+        let sql_val = SqlValue::DoubleQuotedString("MorningMyFriends".to_string());
+        let v = sql_value_to_value(
+            "a",
+            &ConcreteDataType::json_datatype(),
+            &sql_val,
+            None,
+            None,
+        );
+        assert!(v.is_err());
+
+        let sql_val = SqlValue::DoubleQuotedString(r#"{"a":"b"}"#.to_string());
+        let v = sql_value_to_value(
+            "a",
+            &ConcreteDataType::json_datatype(),
+            &sql_val,
+            None,
+            None,
+        )
+        .unwrap();
+        assert_eq!(
+            Value::Binary(Bytes::from(
+                jsonb::parse_value(r#"{"a":"b"}"#.as_bytes())
+                    .unwrap()
+                    .to_vec()
+                    .as_slice()
+            )),
+            v
+        );
     }
 
     #[test]
@@ -1037,6 +1090,36 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_parse_json_to_jsonb() {
+        match parse_string_to_value(
+            "json_col",
+            r#"{"a": "b"}"#.to_string(),
+            &ConcreteDataType::json_datatype(),
+            None,
+        ) {
+            Ok(Value::Binary(b)) => {
+                assert_eq!(
+                    b,
+                    jsonb::parse_value(r#"{"a": "b"}"#.as_bytes())
+                        .unwrap()
+                        .to_vec()
+                );
+            }
+            _ => {
+                unreachable!()
+            }
+        }
+
+        assert!(parse_string_to_value(
+            "json_col",
+            r#"Nicola Kovac is the best rifler in the world"#.to_string(),
+            &ConcreteDataType::json_datatype(),
+            None,
+        )
+        .is_err())
+    }
+
     #[test]
     pub fn test_parse_column_default_constraint() {
         let bool_value = sqlparser::ast::Value::Boolean(true);