diff --git a/Cargo.lock b/Cargo.lock index e2d5e03613..855541bcc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5062,6 +5062,7 @@ dependencies = [ "once_cell", "pin-project-lite", "rand 0.8.5", + "reqwest", "scopeguard", "serde", "serde_json", diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index 1816825bda..33fa6e89f5 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -18,6 +18,7 @@ camino = { workspace = true, features = ["serde1"] } humantime-serde.workspace = true hyper = { workspace = true, features = ["client"] } futures.workspace = true +reqwest.workspace = true serde.workspace = true serde_json.workspace = true tokio = { workspace = true, features = ["sync", "fs", "io-util"] } diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index 32c51bc2ad..c8f94f6e1d 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -8,6 +8,7 @@ use std::io; use std::num::NonZeroU32; use std::pin::Pin; use std::str::FromStr; +use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; @@ -15,7 +16,7 @@ use super::REMOTE_STORAGE_PREFIX_SEPARATOR; use anyhow::Context; use anyhow::Result; use azure_core::request_options::{IfMatchCondition, MaxResults, Metadata, Range}; -use azure_core::{Continuable, RetryOptions}; +use azure_core::{Continuable, RetryOptions, HttpClient, TransportOptions}; use azure_storage::StorageCredentials; use azure_storage_blobs::blob::CopyStatus; use azure_storage_blobs::prelude::ClientBuilder; @@ -81,8 +82,9 @@ impl AzureBlobStorage { }; // we have an outer retry - let builder = ClientBuilder::new(account, credentials).retry(RetryOptions::none()); - + let builder = ClientBuilder::new(account, credentials) + .retry(RetryOptions::none()) + .transport(TransportOptions::new(reqwest_client(small_timeout))); let client = builder.container_client(azure_config.container_name.to_owned()); let max_keys_per_list_response = @@ -261,6 +263,16 @@ impl AzureBlobStorage { } } +fn reqwest_client(timeout: Duration) -> Arc { + let client = reqwest::ClientBuilder::new() + .pool_max_idle_per_host(0) + .read_timeout(timeout) + .connect_timeout(timeout) + .build() + .expect("failed to build `reqwest` client"); + Arc::new(client) +} + fn to_azure_metadata(metadata: StorageMetadata) -> Metadata { let mut res = Metadata::new(); for (k, v) in metadata.0.into_iter() {