mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 16:24:07 +00:00
fix(datatypes): render jsonb stored as JSON text without panics or wrong values (#10197)
* fix(datatypes): decode jsonb before rendering it as text jsonb_to_string serialized the bytes through RawJsonb's Serialize impl, which panics with an out-of-range slice index on JSON text such as "hello world" or " null" (stored by COPY FROM and MySQL text parameters) and on truncated jsonb. Its text fallback was also stricter than the extended parser reads used before. Decode with jsonb::from_slice, as reads did before #10112, then write the value out with serde_json escaping for keys and strings. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * perf(datatypes): keep streaming jsonb serialization in jsonb_to_string Decoding into a jsonb::Value tree allocates per array, object and key on every JSON cell returned over PostgreSQL and MySQL. Go back to serializing the raw bytes with RawJsonb's Serialize impl, and bump jsonb to a rev whose Serialize returns an error instead of panicking on out-of-range payloads (GreptimeTeam/jsonb#2). Bytes that fail to serialize are parsed as JSON text with the extended parser, as jsonb::from_slice does. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * test(datatypes): pin text values that used to read back as false Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: bump jsonb to reject scalar headers with non-zero length bits JSON text over ~2 MB could still pass the payload bounds check and read back as a wrong value (GreptimeTeam/jsonb#2). Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * fix(datatypes): report the text parse error when JSON bytes are neither jsonb nor text Also pin jsonb_to_serde_json on JSON text scalars that used to decode as false. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> * chore: pin jsonb to the merged commit on main GreptimeTeam/jsonb#2 reached main through GreptimeTeam/jsonb#3. Signed-off-by: Dennis Zhuang <killme2008@gmail.com> --------- Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
This commit is contained in:
Generated
+1
-1
@@ -7584,7 +7584,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "jsonb"
|
||||
version = "0.5.6"
|
||||
source = "git+https://github.com/GreptimeTeam/jsonb?rev=dd1e2341e3df35e9de38959328caa7628e81b6b1#dd1e2341e3df35e9de38959328caa7628e81b6b1"
|
||||
source = "git+https://github.com/GreptimeTeam/jsonb?rev=5a6472858d860ac034075c17253f41617bdfa956#5a6472858d860ac034075c17253f41617bdfa956"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"ethnum",
|
||||
|
||||
+1
-1
@@ -176,7 +176,7 @@ hyper-util = "0.1"
|
||||
icu_properties = "2.0.1"
|
||||
indicatif = "0.17"
|
||||
itertools = "0.14"
|
||||
jsonb = { git = "https://github.com/GreptimeTeam/jsonb", rev = "dd1e2341e3df35e9de38959328caa7628e81b6b1", default-features = false, features = [
|
||||
jsonb = { git = "https://github.com/GreptimeTeam/jsonb", rev = "5a6472858d860ac034075c17253f41617bdfa956", default-features = false, features = [
|
||||
"databend",
|
||||
] }
|
||||
lazy_static = "1.4"
|
||||
|
||||
@@ -389,34 +389,33 @@ impl Display for JsonType {
|
||||
|
||||
/// Converts a json type value to string
|
||||
///
|
||||
/// The binary jsonb is serialized directly with `serde_json`, so both object keys and
|
||||
/// string values are escaped by `serde_json`. `RawJsonb::to_string()` is deliberately
|
||||
/// avoided, since it falls back to the lossy raw bytes when serialization fails.
|
||||
/// The binary jsonb is serialized directly with `serde_json`, so object keys and string
|
||||
/// values are escaped by `serde_json`. `RawJsonb::to_string()` is avoided, since it falls
|
||||
/// back to the lossy raw bytes when serialization fails.
|
||||
///
|
||||
/// Some write paths (e.g. `COPY FROM` csv/json/parquet files, or a JSON parameter bound
|
||||
/// as text through the MySQL protocol) currently store the JSON *text* as the column
|
||||
/// bytes instead of binary jsonb, and such data may already be persisted. When the bytes
|
||||
/// are not valid jsonb but are valid JSON text, they are re-encoded as jsonb and then
|
||||
/// serialized the same way, so the output is still escaped by `serde_json`. Anything else
|
||||
/// is reported as an error.
|
||||
/// Some write paths (e.g. `COPY FROM` files, or a JSON parameter bound as text through
|
||||
/// the MySQL protocol) store JSON text in place of binary jsonb. Bytes that fail to
|
||||
/// serialize are parsed as text with the extended parser, as `jsonb::from_slice` does.
|
||||
pub fn jsonb_to_string(val: &[u8]) -> Result<String> {
|
||||
if val.is_empty() {
|
||||
return Ok("".to_string());
|
||||
}
|
||||
match serde_json::to_string(&jsonb::RawJsonb::new(val)) {
|
||||
Ok(s) => Ok(s),
|
||||
Err(e) => match jsonb::parse_value_standard_mode(val) {
|
||||
Ok(json) => serde_json::to_string(&jsonb::RawJsonb::new(&json.to_vec())).map_err(|e| {
|
||||
Err(jsonb_err) => {
|
||||
let value = jsonb::parse_value(val).map_err(|text_err| {
|
||||
InvalidJsonSnafu {
|
||||
value: format!("neither jsonb ({jsonb_err}) nor JSON text ({text_err})"),
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
serde_json::to_string(&jsonb::RawJsonb::new(&value.to_vec())).map_err(|e| {
|
||||
InvalidJsonSnafu {
|
||||
value: e.to_string(),
|
||||
}
|
||||
.build()
|
||||
}),
|
||||
Err(_) => InvalidJsonSnafu {
|
||||
value: e.to_string(),
|
||||
}
|
||||
.fail(),
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -453,6 +452,21 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jsonb_to_serde_json_scalars() {
|
||||
for input in [r#""2026-11-05""#, "1730800000000", "false", "null"] {
|
||||
let expected: serde_json::Value = serde_json::from_str(input).unwrap();
|
||||
let jsonb = parse_string_to_jsonb(input).unwrap();
|
||||
assert_eq!(jsonb_to_serde_json(&jsonb).unwrap(), expected, "{input}");
|
||||
// Stored as JSON text, the first two used to decode as `false`.
|
||||
assert_eq!(
|
||||
jsonb_to_serde_json(input.as_bytes()).unwrap(),
|
||||
expected,
|
||||
"{input}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_jsonb_to_string_escapes_object_keys()
|
||||
-> std::result::Result<(), Box<dyn std::error::Error>> {
|
||||
@@ -514,6 +528,10 @@ mod tests {
|
||||
fn test_jsonb_to_string_rejects_invalid_jsonb() {
|
||||
assert!(jsonb_to_string(&[1, 2, 3, 4]).is_err());
|
||||
assert_eq!(jsonb_to_string(&[]).unwrap(), "");
|
||||
|
||||
let mut truncated = parse_string_to_jsonb(r#"{"k":"value"}"#).unwrap();
|
||||
truncated.pop();
|
||||
assert!(jsonb_to_string(&truncated).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -532,6 +550,20 @@ mod tests {
|
||||
jsonb_to_string(br#"{"a\"b":"c\u0000d"}"#).unwrap(),
|
||||
r#"{"a\"b":"c\u0000d"}"#
|
||||
);
|
||||
// These start with bytes that also read as a jsonb scalar header.
|
||||
assert_eq!(
|
||||
jsonb_to_string(br#""hello world""#).unwrap(),
|
||||
r#""hello world""#
|
||||
);
|
||||
assert_eq!(jsonb_to_string(b" null").unwrap(), "null");
|
||||
// Without the bounds check, these read back as `false`.
|
||||
assert_eq!(
|
||||
jsonb_to_string(br#""2026-11-05""#).unwrap(),
|
||||
r#""2026-11-05""#
|
||||
);
|
||||
assert_eq!(jsonb_to_string(b"1730800000000").unwrap(), "1730800000000");
|
||||
// Extended syntax is accepted, as text was read before.
|
||||
assert_eq!(jsonb_to_string(b"[+1,,2]").unwrap(), "[1,null,2]");
|
||||
assert!(jsonb_to_string(b"{not json").is_err());
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user