hack around azure_core reqwest features

Cargo.lock

@@ -787,7 +787,6 @@ dependencies = [
  "pin-project",
  "quick-xml 0.31.0",
  "rand 0.8.5",
- "reqwest",
  "rustc_version",
  "serde",
  "serde_json",
@@ -5851,7 +5850,6 @@ dependencies = [
  "wasm-bindgen-futures",
  "wasm-streams",
  "web-sys",
- "webpki-roots",
  "winreg",
 ]
 
@@ -6473,15 +6471,12 @@ checksum = "255914a8e53822abd946e2ce8baa41d4cded6b8e938913b7f7b9da5b7ab44335"
 dependencies = [
  "httpdate",
  "reqwest",
- "rustls 0.23.27",
  "sentry-backtrace",
  "sentry-contexts",
  "sentry-core",
  "sentry-panic",
  "sentry-tracing",
  "tokio",
- "ureq",
- "webpki-roots",
 ]
 
 [[package]]
@@ -8151,21 +8146,6 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
 
-[[package]]
-name = "ureq"
-version = "2.10.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a"
-dependencies = [
- "base64 0.22.1",
- "log",
- "once_cell",
- "rustls 0.23.27",
- "rustls-pki-types",
- "url",
- "webpki-roots",
-]
-
 [[package]]
 name = "uring-common"
 version = "0.1.0"
@@ -8542,15 +8522,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "webpki-roots"
-version = "0.26.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009"
-dependencies = [
- "rustls-pki-types",
-]
-
 [[package]]
 name = "whoami"
 version = "1.5.1"
@@ -8861,8 +8832,6 @@ dependencies = [
  "regex-syntax 0.8.2",
  "reqwest",
  "rustls 0.23.27",
- "rustls-pki-types",
- "rustls-webpki 0.103.3",
  "scopeguard",
  "sec1 0.7.3",
  "serde",

Cargo.toml

@@ -160,7 +160,7 @@ prost-types = "0.13.5"
 rand = "0.8"
 redis = { version = "0.29.2", features = ["tokio-rustls-comp", "keep-alive"] }
 regex = "1.10.2"
-reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] }
+reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots"] }
 reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_27"] }
 reqwest-middleware = "0.4"
 reqwest-retry = "0.7"
@@ -174,7 +174,7 @@ scopeguard = "1.1"
 sysinfo = "0.29.2"
 sd-notify = "0.4.1"
 send-future = "0.1.0"
-sentry = { version = "0.37", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
+sentry = { version = "0.37", default-features = false, features = ["backtrace", "contexts", "panic", "reqwest" ] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1"
 serde_path_to_error = "0.1"
@@ -247,10 +247,10 @@ postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", br
 tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" }
 
 ## Azure SDK crates
-azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] }
-azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
-azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
-azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] }
+azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["hmac_rust"] }
+azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = [] }
+azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = [] }
+azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = [] }
 
 ## Local libraries
 compute_api = { version = "0.1", path = "./libs/compute_api/" }

libs/remote_storage/src/azure_blob.rs

@@ -129,7 +129,7 @@ impl AzureBlobStorage {
             .pool_max_idle_per_host(conn_pool_size)
             .build()
             .expect("failed to build `reqwest` client");
-        Arc::new(client)
+        Arc::new(super::azure_reqwest_polyfill::Client(client))
     }
 
     pub fn relative_path_to_name(&self, path: &RemotePath) -> String {

libs/remote_storage/src/azure_reqwest_polyfill.rs

@@ -0,0 +1,96 @@
+use std::{collections::HashMap, str::FromStr};
+
+use azure_core::{
+    Body, HttpClient, Method, Request, Response, StatusCode,
+    error::{Error, ErrorKind, ResultExt},
+    headers::{HeaderName, HeaderValue, Headers},
+};
+use futures::TryStreamExt;
+use tracing::{debug, warn};
+
+#[derive(Debug)]
+pub struct Client(pub reqwest::Client);
+
+#[async_trait::async_trait]
+impl HttpClient for Client {
+    async fn execute_request(&self, request: &Request) -> azure_core::Result<Response> {
+        let url = request.url().clone();
+        let method = request.method();
+        let mut req = self.0.request(try_from_method(*method)?, url.clone());
+
+        for (name, value) in request.headers().iter() {
+            req = req.header(name.as_str(), value.as_str());
+        }
+
+        let req = match request.body().clone() {
+            Body::Bytes(bytes) => req.body(bytes),
+            Body::SeekableStream(seekable_stream) => {
+                req.body(reqwest::Body::wrap_stream(seekable_stream))
+            }
+        };
+
+        let reqwest_request = req
+            .build()
+            .context(ErrorKind::Other, "failed to build `reqwest` request")?;
+
+        debug!("performing request {method} '{url}' with `reqwest`");
+        let rsp = self
+            .0
+            .execute(reqwest_request)
+            .await
+            .context(ErrorKind::Io, "failed to execute `reqwest` request")?;
+
+        let status = rsp.status();
+        let headers = to_headers(rsp.headers());
+
+        let body = Box::pin(
+            rsp.bytes_stream()
+                .map_err(|error| Error::new(ErrorKind::Io, error)),
+        );
+
+        Ok(Response::new(try_from_status(status)?, headers, body))
+    }
+}
+
+fn to_headers(map: &reqwest::header::HeaderMap) -> Headers {
+    let map = map
+        .iter()
+        .filter_map(|(k, v)| {
+            let key = k.as_str();
+            if let Ok(value) = v.to_str() {
+                Some((
+                    HeaderName::from(key.to_owned()),
+                    HeaderValue::from(value.to_owned()),
+                ))
+            } else {
+                warn!("header value for `{key}` is not utf8");
+                None
+            }
+        })
+        .collect::<HashMap<_, _>>();
+    Headers::from(map)
+}
+
+fn try_from_method(method: Method) -> azure_core::Result<reqwest::Method> {
+    match method {
+        Method::Connect => Ok(reqwest::Method::CONNECT),
+        Method::Delete => Ok(reqwest::Method::DELETE),
+        Method::Get => Ok(reqwest::Method::GET),
+        Method::Head => Ok(reqwest::Method::HEAD),
+        Method::Options => Ok(reqwest::Method::OPTIONS),
+        Method::Patch => Ok(reqwest::Method::PATCH),
+        Method::Post => Ok(reqwest::Method::POST),
+        Method::Put => Ok(reqwest::Method::PUT),
+        Method::Trace => Ok(reqwest::Method::TRACE),
+        _ => reqwest::Method::from_str(method.as_ref()).map_kind(ErrorKind::DataConversion),
+    }
+}
+
+fn try_from_status(status: reqwest::StatusCode) -> azure_core::Result<StatusCode> {
+    let status = u16::from(status);
+    StatusCode::try_from(status).map_err(|_| {
+        Error::with_message(ErrorKind::DataConversion, || {
+            format!("invalid status code {status}")
+        })
+    })
+}

libs/remote_storage/src/lib.rs

@@ -10,6 +10,7 @@
 #![deny(clippy::undocumented_unsafe_blocks)]
 
 mod azure_blob;
+mod azure_reqwest_polyfill;
 mod config;
 mod error;
 mod local_fs;

storage_scrubber/Cargo.toml

@@ -29,7 +29,7 @@ storage_controller_client.workspace = true
 
 tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
 chrono = { workspace = true, default-features = false, features = ["clock", "serde"] }
-reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "json"] }
+reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots", "json"] }
 
 pageserver = { path = "../pageserver" }
 pageserver_api = { path = "../libs/pageserver_api" }

workspace_hack/Cargo.toml

@@ -78,10 +78,8 @@ rand = { version = "0.8", features = ["small_rng"] }
 regex = { version = "1" }
 regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
 regex-syntax = { version = "0.8" }
-reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "rustls-tls-native-roots", "stream"] }
+reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls-native-roots", "stream"] }
 rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] }
-rustls-pki-types = { version = "1", features = ["std"] }
-rustls-webpki = { version = "0.103", default-features = false, features = ["ring", "std"] }
 scopeguard = { version = "1" }
 sec1 = { version = "0.7", features = ["pem", "serde", "std", "subtle"] }
 serde = { version = "1", features = ["alloc", "derive"] }