diff --git a/Cargo.lock b/Cargo.lock index e9004748ae..e2d5e03613 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,77 +770,74 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fd680c0d0424a518229b1150922f92653ba2ac933aa000abc8bf1ca08105f7" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "async-trait", - "base64 0.21.1", + "base64 0.22.1", "bytes", "dyn-clone", "futures", "getrandom 0.2.11", "hmac", "http-types", - "log", "once_cell", "paste", "pin-project", "quick-xml 0.31.0", "rand 0.8.5", - "reqwest 0.11.19", + "reqwest", "rustc_version", "serde", "serde_json", "sha2", "time", + "tracing", "url", "uuid", ] [[package]] name = "azure_identity" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d2060f5b2e1c664026ca4edd561306c473be887c1f7a81f10bf06f9b71c63f" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "async-lock", "async-trait", "azure_core", "futures", - "log", "oauth2", "pin-project", "serde", "time", + "tokio", + "tracing", "url", "uuid", ] [[package]] name = "azure_storage" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d3da73bfa09350e1bd6ae2a260806fcf90048c7e78cd2d8f88be60b19a7266" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "RustyXML", "async-lock", "async-trait", "azure_core", "bytes", - "log", "serde", "serde_derive", "time", + "tracing", "url", "uuid", ] [[package]] name = "azure_storage_blobs" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "149c21834a4105d761e3dd33d91c2a3064acc05a3c978848ea8089102ae45c94" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "RustyXML", "azure_core", @@ -848,20 +845,19 @@ dependencies = [ "azure_svc_blobstorage", "bytes", "futures", - "log", "serde", "serde_derive", "serde_json", "time", + "tracing", "url", "uuid", ] [[package]] name = "azure_svc_blobstorage" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88c888b7bf522d5405218b8613bf0fae7ddaae6ef3bf4ad42ae005993c96ab8b" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "azure_core", "bytes", @@ -1287,7 +1283,7 @@ dependencies = [ "prometheus", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "rlimit", "rust-ini", "serde", @@ -1395,7 +1391,7 @@ dependencies = [ "postgres_backend", "postgres_connection", "regex", - "reqwest 0.12.4", + "reqwest", "safekeeper_api", "scopeguard", "serde", @@ -1904,15 +1900,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "encoding_rs" -version = "0.8.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" -dependencies = [ - "cfg-if", -] - [[package]] name = "enum-map" version = "2.5.0" @@ -3709,7 +3696,7 @@ dependencies = [ "bytes", "http 1.1.0", "opentelemetry", - "reqwest 0.12.4", + "reqwest", ] [[package]] @@ -3726,7 +3713,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.12.4", + "reqwest", "thiserror", ] @@ -3935,7 +3922,7 @@ dependencies = [ "range-set-blaze", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "rpds", "scopeguard", "send-future", @@ -3988,7 +3975,7 @@ dependencies = [ "postgres_ffi", "rand 0.8.5", "remote_storage", - "reqwest 0.12.4", + "reqwest", "serde", "serde_json", "serde_with", @@ -4008,7 +3995,7 @@ dependencies = [ "futures", "pageserver_api", "postgres", - "reqwest 0.12.4", + "reqwest", "serde", "thiserror", "tokio", @@ -4725,7 +4712,7 @@ dependencies = [ "redis", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "reqwest-middleware", "reqwest-retry", "reqwest-tracing", @@ -5088,47 +5075,6 @@ dependencies = [ "utils", ] -[[package]] -name = "reqwest" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" -dependencies = [ - "base64 0.21.1", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.30", - "hyper-rustls 0.24.0", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls 0.21.12", - "rustls-pemfile 1.0.2", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-rustls 0.24.0", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams 0.3.0", - "web-sys", - "webpki-roots 0.25.2", - "winreg 0.50.0", -] - [[package]] name = "reqwest" version = "0.12.4" @@ -5168,10 +5114,10 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.4.0", + "wasm-streams", "web-sys", "webpki-roots 0.26.1", - "winreg 0.52.0", + "winreg", ] [[package]] @@ -5183,7 +5129,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.1.0", - "reqwest 0.12.4", + "reqwest", "serde", "thiserror", "tower-service", @@ -5202,7 +5148,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "parking_lot 0.11.2", - "reqwest 0.12.4", + "reqwest", "reqwest-middleware", "retry-policies", "thiserror", @@ -5223,7 +5169,7 @@ dependencies = [ "http 1.1.0", "matchit 0.8.2", "opentelemetry", - "reqwest 0.12.4", + "reqwest", "reqwest-middleware", "tracing", "tracing-opentelemetry", @@ -5587,7 +5533,7 @@ dependencies = [ "rand 0.8.5", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "safekeeper_api", "scopeguard", "sd-notify", @@ -5743,7 +5689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00421ed8fa0c995f07cde48ba6c89e80f2b312f74ff637326f392fbfd23abe02" dependencies = [ "httpdate", - "reqwest 0.12.4", + "reqwest", "rustls 0.21.12", "sentry-backtrace", "sentry-contexts", @@ -6198,7 +6144,7 @@ dependencies = [ "postgres_connection", "r2d2", "rand 0.8.5", - "reqwest 0.12.4", + "reqwest", "routerify", "scopeguard", "serde", @@ -6218,7 +6164,7 @@ name = "storage_controller_client" version = "0.1.0" dependencies = [ "pageserver_client", - "reqwest 0.12.4", + "reqwest", "serde", "workspace_hack", ] @@ -6245,7 +6191,7 @@ dependencies = [ "pageserver_api", "postgres_ffi", "remote_storage", - "reqwest 0.12.4", + "reqwest", "rustls 0.23.18", "rustls-native-certs 0.8.0", "serde", @@ -6274,7 +6220,7 @@ dependencies = [ "humantime", "pageserver_api", "pageserver_client", - "reqwest 0.12.4", + "reqwest", "serde_json", "storage_controller_client", "tokio", @@ -7514,19 +7460,6 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" -[[package]] -name = "wasm-streams" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasm-streams" version = "0.4.0" @@ -7792,16 +7725,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "winreg" version = "0.52.0" @@ -7876,7 +7799,7 @@ dependencies = [ "regex", "regex-automata 0.4.3", "regex-syntax 0.8.2", - "reqwest 0.12.4", + "reqwest", "rustls 0.23.18", "scopeguard", "serde", diff --git a/Cargo.toml b/Cargo.toml index a35823e0c2..0654c25a3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,10 +51,6 @@ anyhow = { version = "1.0", features = ["backtrace"] } arc-swap = "1.6" async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] } atomic-take = "1.1.0" -azure_core = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } -azure_identity = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] } -azure_storage = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] } -azure_storage_blobs = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] } flate2 = "1.0.26" async-stream = "0.3" async-trait = "0.1" @@ -216,6 +212,12 @@ postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" } tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" } +## Azure SDK crates +azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } +azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } +azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } +azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } + ## Local libraries compute_api = { version = "0.1", path = "./libs/compute_api/" } consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" } diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index 8d1962fa29..a1d7569140 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -8,15 +8,14 @@ use std::io; use std::num::NonZeroU32; use std::pin::Pin; use std::str::FromStr; -use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; use super::REMOTE_STORAGE_PREFIX_SEPARATOR; +use anyhow::Context; use anyhow::Result; use azure_core::request_options::{IfMatchCondition, MaxResults, Metadata, Range}; use azure_core::{Continuable, RetryOptions}; -use azure_identity::DefaultAzureCredential; use azure_storage::StorageCredentials; use azure_storage_blobs::blob::CopyStatus; use azure_storage_blobs::prelude::ClientBuilder; @@ -76,8 +75,9 @@ impl AzureBlobStorage { let credentials = if let Ok(access_key) = env::var("AZURE_STORAGE_ACCESS_KEY") { StorageCredentials::access_key(account.clone(), access_key) } else { - let token_credential = DefaultAzureCredential::default(); - StorageCredentials::token_credential(Arc::new(token_credential)) + let token_credential = azure_identity::create_default_credential() + .context("trying to obtain Azure default credentials")?; + StorageCredentials::token_credential(token_credential) }; // we have an outer retry