fix(datatypes): render jsonb stored as JSON text without panics or wrong values (#10197)

* fix(datatypes): decode jsonb before rendering it as text

jsonb_to_string serialized the bytes through RawJsonb's Serialize impl,
which panics with an out-of-range slice index on JSON text such as
"hello world" or "        null" (stored by COPY FROM and MySQL text
parameters) and on truncated jsonb. Its text fallback was also stricter than
the extended parser reads used before.

Decode with jsonb::from_slice, as reads did before #10112, then write the
value out with serde_json escaping for keys and strings.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* perf(datatypes): keep streaming jsonb serialization in jsonb_to_string

Decoding into a jsonb::Value tree allocates per array, object and key on
every JSON cell returned over PostgreSQL and MySQL. Go back to serializing
the raw bytes with RawJsonb's Serialize impl, and bump jsonb to a rev whose
Serialize returns an error instead of panicking on out-of-range payloads
(GreptimeTeam/jsonb#2). Bytes that fail to serialize are parsed as JSON
text with the extended parser, as jsonb::from_slice does.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* test(datatypes): pin text values that used to read back as false

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: bump jsonb to reject scalar headers with non-zero length bits

JSON text over ~2 MB could still pass the payload bounds check and read back as a wrong value (GreptimeTeam/jsonb#2).

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* fix(datatypes): report the text parse error when JSON bytes are neither jsonb nor text

Also pin jsonb_to_serde_json on JSON text scalars that used to decode as false.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* chore: pin jsonb to the merged commit on main

GreptimeTeam/jsonb#2 reached main through GreptimeTeam/jsonb#3.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

---------

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
This commit is contained in:
dennis zhuang
2026-11-11 03:46:56 +00:00
committed by GitHub
parent 97c8136c3c
commit ddc16bafe8
3 changed files with 51 additions and 19 deletions
Generated
+1 -1
View File
@@ -7584,7 +7584,7 @@ dependencies = [
[[package]]
name = "jsonb"
version = "0.5.6"
source = "git+https://github.com/GreptimeTeam/jsonb?rev=dd1e2341e3df35e9de38959328caa7628e81b6b1#dd1e2341e3df35e9de38959328caa7628e81b6b1"
source = "git+https://github.com/GreptimeTeam/jsonb?rev=5a6472858d860ac034075c17253f41617bdfa956#5a6472858d860ac034075c17253f41617bdfa956"
dependencies = [
"byteorder",
"ethnum",
+1 -1
View File
@@ -176,7 +176,7 @@ hyper-util = "0.1"
icu_properties = "2.0.1"
indicatif = "0.17"
itertools = "0.14"
jsonb = { git = "https://github.com/GreptimeTeam/jsonb", rev = "dd1e2341e3df35e9de38959328caa7628e81b6b1", default-features = false, features = [
jsonb = { git = "https://github.com/GreptimeTeam/jsonb", rev = "5a6472858d860ac034075c17253f41617bdfa956", default-features = false, features = [
"databend",
] }
lazy_static = "1.4"
+49 -17
View File
@@ -389,34 +389,33 @@ impl Display for JsonType {
/// Converts a json type value to string
///
/// The binary jsonb is serialized directly with `serde_json`, so both object keys and
/// string values are escaped by `serde_json`. `RawJsonb::to_string()` is deliberately
/// avoided, since it falls back to the lossy raw bytes when serialization fails.
/// The binary jsonb is serialized directly with `serde_json`, so object keys and string
/// values are escaped by `serde_json`. `RawJsonb::to_string()` is avoided, since it falls
/// back to the lossy raw bytes when serialization fails.
///
/// Some write paths (e.g. `COPY FROM` csv/json/parquet files, or a JSON parameter bound
/// as text through the MySQL protocol) currently store the JSON *text* as the column
/// bytes instead of binary jsonb, and such data may already be persisted. When the bytes
/// are not valid jsonb but are valid JSON text, they are re-encoded as jsonb and then
/// serialized the same way, so the output is still escaped by `serde_json`. Anything else
/// is reported as an error.
/// Some write paths (e.g. `COPY FROM` files, or a JSON parameter bound as text through
/// the MySQL protocol) store JSON text in place of binary jsonb. Bytes that fail to
/// serialize are parsed as text with the extended parser, as `jsonb::from_slice` does.
pub fn jsonb_to_string(val: &[u8]) -> Result<String> {
if val.is_empty() {
return Ok("".to_string());
}
match serde_json::to_string(&jsonb::RawJsonb::new(val)) {
Ok(s) => Ok(s),
Err(e) => match jsonb::parse_value_standard_mode(val) {
Ok(json) => serde_json::to_string(&jsonb::RawJsonb::new(&json.to_vec())).map_err(|e| {
Err(jsonb_err) => {
let value = jsonb::parse_value(val).map_err(|text_err| {
InvalidJsonSnafu {
value: format!("neither jsonb ({jsonb_err}) nor JSON text ({text_err})"),
}
.build()
})?;
serde_json::to_string(&jsonb::RawJsonb::new(&value.to_vec())).map_err(|e| {
InvalidJsonSnafu {
value: e.to_string(),
}
.build()
}),
Err(_) => InvalidJsonSnafu {
value: e.to_string(),
}
.fail(),
},
})
}
}
}
@@ -453,6 +452,21 @@ mod tests {
Ok(())
}
#[test]
fn test_jsonb_to_serde_json_scalars() {
for input in [r#""2026-11-05""#, "1730800000000", "false", "null"] {
let expected: serde_json::Value = serde_json::from_str(input).unwrap();
let jsonb = parse_string_to_jsonb(input).unwrap();
assert_eq!(jsonb_to_serde_json(&jsonb).unwrap(), expected, "{input}");
// Stored as JSON text, the first two used to decode as `false`.
assert_eq!(
jsonb_to_serde_json(input.as_bytes()).unwrap(),
expected,
"{input}"
);
}
}
#[test]
fn test_jsonb_to_string_escapes_object_keys()
-> std::result::Result<(), Box<dyn std::error::Error>> {
@@ -514,6 +528,10 @@ mod tests {
fn test_jsonb_to_string_rejects_invalid_jsonb() {
assert!(jsonb_to_string(&[1, 2, 3, 4]).is_err());
assert_eq!(jsonb_to_string(&[]).unwrap(), "");
let mut truncated = parse_string_to_jsonb(r#"{"k":"value"}"#).unwrap();
truncated.pop();
assert!(jsonb_to_string(&truncated).is_err());
}
#[test]
@@ -532,6 +550,20 @@ mod tests {
jsonb_to_string(br#"{"a\"b":"c\u0000d"}"#).unwrap(),
r#"{"a\"b":"c\u0000d"}"#
);
// These start with bytes that also read as a jsonb scalar header.
assert_eq!(
jsonb_to_string(br#""hello world""#).unwrap(),
r#""hello world""#
);
assert_eq!(jsonb_to_string(b" null").unwrap(), "null");
// Without the bounds check, these read back as `false`.
assert_eq!(
jsonb_to_string(br#""2026-11-05""#).unwrap(),
r#""2026-11-05""#
);
assert_eq!(jsonb_to_string(b"1730800000000").unwrap(), "1730800000000");
// Extended syntax is accepted, as text was read before.
assert_eq!(jsonb_to_string(b"[+1,,2]").unwrap(), "[1,null,2]");
assert!(jsonb_to_string(b"{not json").is_err());
}