storcon: reuse reqwest http client (#11327)

## Problem

- Part of https://github.com/neondatabase/neon/issues/11113
- Building a new `reqwest::Client` for every request is expensive
because it parses CA certs under the hood. It's noticeable in storcon's
flamegraph.

## Summary of changes
- Reuse one `reqwest::Client` for all API calls to avoid parsing CA
certificates every time.

storage_controller/src/node.rs

@@ -7,7 +7,7 @@ use pageserver_api::controller_api::{
 };
 use pageserver_api::shard::TenantShardId;
 use pageserver_client::mgmt_api;
-use reqwest::{Certificate, StatusCode};
+use reqwest::StatusCode;
 use serde::Serialize;
 use tokio_util::sync::CancellationToken;
 use utils::backoff;
@@ -280,8 +280,8 @@ impl Node {
     pub(crate) async fn with_client_retries<T, O, F>(
         &self,
         mut op: O,
+        http_client: &reqwest::Client,
         jwt: &Option<String>,
-        ssl_ca_cert: &Option<Certificate>,
         warn_threshold: u32,
         max_retries: u32,
         timeout: Duration,
@@ -300,24 +300,13 @@ impl Node {
                 | ApiError(StatusCode::REQUEST_TIMEOUT, _) => false,
                 ApiError(_, _) => true,
                 Cancelled => true,
-                CreateClient(_) => true,
+                Timeout(_) => false,
             }
         }
 
-        // TODO: refactor PageserverClient and with_client_retires (#11113).
-        let mut http_client = reqwest::ClientBuilder::new().timeout(timeout);
-        if let Some(ssl_ca_cert) = ssl_ca_cert.as_ref() {
-            http_client = http_client.add_root_certificate(ssl_ca_cert.clone())
-        }
-
-        let http_client = match http_client.build() {
-            Ok(http_client) => http_client,
-            Err(err) => return Some(Err(mgmt_api::Error::CreateClient(err))),
-        };
-
         backoff::retry(
             || {
-                let client = PageserverClient::from_client(
+                let client = PageserverClient::new(
                     self.get_id(),
                     http_client.clone(),
                     self.base_url(),
@@ -326,11 +315,14 @@ impl Node {
 
                 let node_cancel_fut = self.cancel.cancelled();
 
-                let op_fut = op(client);
+                let op_fut = tokio::time::timeout(timeout, op(client));
 
                 async {
                     tokio::select! {
-                        r = op_fut=> {r},
+                        r = op_fut => match r {
+                            Ok(r) => r,
+                            Err(e) => Err(mgmt_api::Error::Timeout(format!("{e}"))),
+                        },
                         _ = node_cancel_fut => {
                         Err(mgmt_api::Error::Cancelled)
                     }}