diff --git a/Cargo.lock b/Cargo.lock index fc5ea07564..43f7bc3874 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4100,6 +4100,7 @@ dependencies = [ "num-traits", "ordered-float 4.6.0", "paste", + "regex", "serde", "serde_json", "snafu 0.8.6", diff --git a/src/datatypes/Cargo.toml b/src/datatypes/Cargo.toml index 8eb642f5d6..3bf2a5d8ee 100644 --- a/src/datatypes/Cargo.toml +++ b/src/datatypes/Cargo.toml @@ -28,6 +28,7 @@ num = "0.4" num-traits = "0.2" ordered-float.workspace = true paste.workspace = true +regex.workspace = true serde.workspace = true serde_json.workspace = true snafu.workspace = true diff --git a/src/datatypes/src/types/json_type.rs b/src/datatypes/src/types/json_type.rs index 7e8d69252e..c8a3780217 100644 --- a/src/datatypes/src/types/json_type.rs +++ b/src/datatypes/src/types/json_type.rs @@ -15,10 +15,11 @@ use std::collections::BTreeMap; use std::fmt::{Debug, Display, Formatter}; use std::str::FromStr; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use arrow::datatypes::DataType as ArrowDataType; use common_base::bytes::Bytes; +use regex::{Captures, Regex}; use serde::{Deserialize, Serialize}; use snafu::ResultExt; @@ -401,8 +402,52 @@ pub fn jsonb_to_string(val: &[u8]) -> Result { /// Converts a json type value to serde_json::Value pub fn jsonb_to_serde_json(val: &[u8]) -> Result { let json_string = jsonb_to_string(val)?; - serde_json::Value::from_str(json_string.as_str()) - .context(DeserializeSnafu { json: json_string }) + jsonb_string_to_serde_value(&json_string) +} + +fn jsonb_string_to_serde_value(json: &str) -> Result { + match serde_json::Value::from_str(json) { + Ok(v) => Ok(v), + Err(_) => { + // If above deserialization is failed, the JSON string might contain some Rust chars + // that are somehow mis-represented as Unicode code point literal. For example, + // "\u{fe0f}". We have to convert them to JSON compatible format, like "\uFE0F", then + // try to deserialize the JSON string again. + + static UNICODE_CODE_POINT_PATTERN: LazyLock = LazyLock::new(|| { + // Match literal "\u{...}" sequences, capturing 1–6 (code point range) hex digits + // inside braces. + Regex::new(r"\\u\{([0-9a-fA-F]{1,6})}").unwrap_or_else(|e| panic!("{}", e)) + }); + + let v = UNICODE_CODE_POINT_PATTERN.replace_all(json, |caps: &Captures| { + // Extract the hex payload (without braces) and parse to a code point. + let hex = &caps[1]; + let Ok(code) = u32::from_str_radix(hex, 16) else { + return json.to_string(); + }; + + if code <= 0xFFFF { + // Basic Multilingual Plane: JSON can represent this directly as \uXXXX. + format!("\\u{:04X}", code) + } else { + // Supplementary planes: JSON needs UTF-16 surrogate pairs. + // Convert the code point to a 20-bit value. + let code = code - 0x10000; + + // High surrogate: top 10 bits, offset by 0xD800. + let high = 0xD800 + ((code >> 10) & 0x3FF); + + // Low surrogate: bottom 10 bits, offset by 0xDC00. + let low = 0xDC00 + (code & 0x3FF); + + // Emit two \uXXXX escapes in sequence. + format!("\\u{:04X}\\u{:04X}", high, low) + } + }); + serde_json::Value::from_str(&v).context(DeserializeSnafu { json }) + } + } } /// Parses a string to a json type value @@ -417,6 +462,26 @@ mod tests { use super::*; use crate::json::JsonStructureSettings; + #[test] + fn test_jsonb_string_to_serde_value() -> Result<()> { + let cases = vec![ + (r#"{"data": "simple ascii"}"#, r#"{"data":"simple ascii"}"#), + ( + r#"{"data": "Greek sigma: \u{03a3}"}"#, + r#"{"data":"Greek sigma: Σ"}"#, + ), + ( + r#"{"data": "Joker card: \u{1f0df}"}"#, + r#"{"data":"Joker card: 🃟"}"#, + ), + ]; + for (input, expect) in cases { + let v = jsonb_string_to_serde_value(input)?; + assert_eq!(v.to_string(), expect); + } + Ok(()) + } + #[test] fn test_json_type_include() { fn test(this: &JsonNativeType, that: &JsonNativeType, expected: bool) {