diff --git a/Cargo.lock b/Cargo.lock index c078510129..6856b9e3ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2770,7 +2770,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c#f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" dependencies = [ "bytes", "fallible-iterator", @@ -2783,7 +2783,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c#f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" dependencies = [ "native-tls", "tokio", @@ -2794,7 +2794,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c#f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" dependencies = [ "base64 0.20.0", "byteorder", @@ -2812,7 +2812,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c#f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" dependencies = [ "bytes", "fallible-iterator", @@ -4272,7 +4272,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c#f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" dependencies = [ "async-trait", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index d7bffe67e1..dc34705f8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,11 +140,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" } tokio-tar = { git = "https://github.com/neondatabase/tokio-tar.git", rev="404df61437de0feef49ba2ccdbdd94eb8ad6e142" } ## Other git libraries @@ -180,7 +180,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="f6ec31df3bcce89cb34f300f17c8a8c031c5ee8c" } # Changes the MAX_THREADS limit from 4096 to 32768. # This is a temporary workaround for using tracing from many threads in safekeepers code, diff --git a/proxy/src/http/sql_over_http.rs b/proxy/src/http/sql_over_http.rs index 417af1e531..050f00dd7d 100644 --- a/proxy/src/http/sql_over_http.rs +++ b/proxy/src/http/sql_over_http.rs @@ -33,17 +33,20 @@ static HEADER_VALUE_TRUE: HeaderValue = HeaderValue::from_static("true"); // Convert json non-string types to strings, so that they can be passed to Postgres // as parameters. // -fn json_to_pg_text(json: Vec) -> Result, serde_json::Error> { +fn json_to_pg_text(json: Vec) -> Result>, serde_json::Error> { json.iter() .map(|value| { match value { - Value::Null => serde_json::to_string(value), - Value::Bool(_) => serde_json::to_string(value), - Value::Number(_) => serde_json::to_string(value), - Value::Object(_) => serde_json::to_string(value), + // special care for nulls + Value::Null => Ok(None), - // no need to escape - Value::String(s) => Ok(s.to_string()), + // convert to text with escaping + Value::Bool(_) => serde_json::to_string(value).map(Some), + Value::Number(_) => serde_json::to_string(value).map(Some), + Value::Object(_) => serde_json::to_string(value).map(Some), + + // avoid escaping here, as we pass this as a parameter + Value::String(s) => Ok(Some(s.to_string())), // special care for arrays Value::Array(_) => json_array_to_pg_array(value), @@ -60,25 +63,29 @@ fn json_to_pg_text(json: Vec) -> Result, serde_json::Error> { // // Example of the same escaping in node-postgres: packages/pg/lib/utils.js // -fn json_array_to_pg_array(value: &Value) -> Result { +fn json_array_to_pg_array(value: &Value) -> Result, serde_json::Error> { match value { - // same - Value::Null => serde_json::to_string(value), - Value::Bool(_) => serde_json::to_string(value), - Value::Number(_) => serde_json::to_string(value), - Value::Object(_) => serde_json::to_string(value), + // special care for nulls + Value::Null => Ok(None), - // now needs to be escaped, as it is part of the array - Value::String(_) => serde_json::to_string(value), + // convert to text with escaping + Value::Bool(_) => serde_json::to_string(value).map(Some), + Value::Number(_) => serde_json::to_string(value).map(Some), + Value::Object(_) => serde_json::to_string(value).map(Some), + + // here string needs to be escaped, as it is part of the array + Value::String(_) => serde_json::to_string(value).map(Some), // recurse into array Value::Array(arr) => { let vals = arr .iter() .map(json_array_to_pg_array) + .map(|r| r.map(|v| v.unwrap_or_else(|| "NULL".to_string()))) .collect::, _>>()? .join(","); - Ok(format!("{{{}}}", vals)) + + Ok(Some(format!("{{{}}}", vals))) } } } @@ -335,10 +342,6 @@ pub fn pg_text_row_to_json( // pub fn pg_text_to_json(pg_value: Option<&str>, pg_type: &Type) -> Result { if let Some(val) = pg_value { - if val == "NULL" { - return Ok(Value::Null); - } - if let Kind::Array(elem_type) = pg_type.kind() { return pg_array_parse(val, elem_type); } @@ -400,6 +403,27 @@ fn _pg_array_parse( } } + fn push_checked( + entry: &mut String, + entries: &mut Vec, + elem_type: &Type, + ) -> Result<(), anyhow::Error> { + if !entry.is_empty() { + // While in usual postgres response we get nulls as None and everything else + // as Some(&str), in arrays we get NULL as unquoted 'NULL' string (while + // string with value 'NULL' will be represented by '"NULL"'). So catch NULLs + // here while we have quotation info and convert them to None. + if entry == "NULL" { + entries.push(pg_text_to_json(None, elem_type)?); + } else { + entries.push(pg_text_to_json(Some(entry), elem_type)?); + } + entry.clear(); + } + + Ok(()) + } + while let Some((mut i, mut c)) = pg_array_chr.next() { let mut escaped = false; @@ -422,9 +446,7 @@ fn _pg_array_parse( '}' => { level -= 1; if level == 0 { - if !entry.is_empty() { - entries.push(pg_text_to_json(Some(&entry), elem_type)?); - } + push_checked(&mut entry, &mut entries, elem_type)?; if nested { return Ok((Value::Array(entries), i)); } @@ -432,17 +454,15 @@ fn _pg_array_parse( } '"' if !escaped => { if quote { - // push even if empty + // end of quoted string, so push it manually without any checks + // for emptiness or nulls entries.push(pg_text_to_json(Some(&entry), elem_type)?); - entry = String::new(); + entry.clear(); } quote = !quote; } ',' if !quote => { - if !entry.is_empty() { - entries.push(pg_text_to_json(Some(&entry), elem_type)?); - entry = String::new(); - } + push_checked(&mut entry, &mut entries, elem_type)?; } _ => { entry.push(c); @@ -466,30 +486,35 @@ mod tests { fn test_atomic_types_to_pg_params() { let json = vec![Value::Bool(true), Value::Bool(false)]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["true", "false"]); + assert_eq!( + pg_params, + vec![Some("true".to_owned()), Some("false".to_owned())] + ); let json = vec![Value::Number(serde_json::Number::from(42))]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["42"]); + assert_eq!(pg_params, vec![Some("42".to_owned())]); let json = vec![Value::String("foo\"".to_string())]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["foo\""]); + assert_eq!(pg_params, vec![Some("foo\"".to_owned())]); let json = vec![Value::Null]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["null"]); + assert_eq!(pg_params, vec![None]); } #[test] fn test_json_array_to_pg_array() { // atoms and escaping - let json = "[true, false, null, 42, \"foo\", \"bar\\\"-\\\\\"]"; + let json = "[true, false, null, \"NULL\", 42, \"foo\", \"bar\\\"-\\\\\"]"; let json: Value = serde_json::from_str(json).unwrap(); let pg_params = json_to_pg_text(vec![json]).unwrap(); assert_eq!( pg_params, - vec!["{true,false,null,42,\"foo\",\"bar\\\"-\\\\\"}"] + vec![Some( + "{true,false,NULL,\"NULL\",42,\"foo\",\"bar\\\"-\\\\\"}".to_owned() + )] ); // nested arrays @@ -498,7 +523,9 @@ mod tests { let pg_params = json_to_pg_text(vec![json]).unwrap(); assert_eq!( pg_params, - vec!["{{true,false},{null,42},{\"foo\",\"bar\\\"-\\\\\"}}"] + vec![Some( + "{{true,false},{NULL,42},{\"foo\",\"bar\\\"-\\\\\"}}".to_owned() + )] ); }