diff --git a/Cargo.lock b/Cargo.lock index e2d5e03613..33909e39e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -771,7 +771,7 @@ dependencies = [ [[package]] name = "azure_core" version = "0.21.0" -source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=arpad%2Fblob_batch#e34e9d277391a394dce6bd016db8d50991462f72" dependencies = [ "async-trait", "base64 0.22.1", @@ -800,7 +800,7 @@ dependencies = [ [[package]] name = "azure_identity" version = "0.21.0" -source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=arpad%2Fblob_batch#e34e9d277391a394dce6bd016db8d50991462f72" dependencies = [ "async-lock", "async-trait", @@ -819,7 +819,7 @@ dependencies = [ [[package]] name = "azure_storage" version = "0.21.0" -source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=arpad%2Fblob_batch#e34e9d277391a394dce6bd016db8d50991462f72" dependencies = [ "RustyXML", "async-lock", @@ -837,7 +837,7 @@ dependencies = [ [[package]] name = "azure_storage_blobs" version = "0.21.0" -source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=arpad%2Fblob_batch#e34e9d277391a394dce6bd016db8d50991462f72" dependencies = [ "RustyXML", "azure_core", @@ -857,7 +857,7 @@ dependencies = [ [[package]] name = "azure_svc_blobstorage" version = "0.21.0" -source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=arpad%2Fblob_batch#e34e9d277391a394dce6bd016db8d50991462f72" dependencies = [ "azure_core", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 0654c25a3d..60a4cd5b82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -213,10 +213,10 @@ postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", br tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" } ## Azure SDK crates -azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } -azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } -azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } -azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } +azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "arpad/blob_batch", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } +azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "arpad/blob_batch", default-features = false, features = ["enable_reqwest_rustls"] } +azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "arpad/blob_batch", default-features = false, features = ["enable_reqwest_rustls"] } +azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "arpad/blob_batch", default-features = false, features = ["enable_reqwest_rustls"] } ## Local libraries compute_api = { version = "0.1", path = "./libs/compute_api/" } diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index a1d7569140..1626274ef1 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -556,7 +556,7 @@ impl RemoteStorage for AzureBlobStorage { let op = async { // TODO batch requests are not supported by the SDK // https://github.com/Azure/azure-sdk-for-rust/issues/1068 - for path in paths { + for path_chunk in paths.chunks(256) { #[derive(Debug)] enum AzureOrTimeout { AzureError(azure_core::Error), @@ -572,13 +572,20 @@ impl RemoteStorage for AzureBlobStorage { let max_retries = 5; backoff::retry( || async { - let blob_client = self.client.blob_client(self.relative_path_to_name(path)); + let mut batch_client = self.client.blob_batch(); + for path in path_chunk { + batch_client = match batch_client.delete(self.relative_path_to_name(path)) { + Ok(batch_client) => batch_client, + Err(e) => return Err(AzureOrTimeout::AzureError(e)), + }; + } - let request = blob_client.delete().into_future(); + let request = batch_client.into_future(); let res = tokio::time::timeout(self.timeout, request).await; match res { + // TODO: validate that all deletions were successful Ok(Ok(_v)) => Ok(()), Ok(Err(azure_err)) => { if let Some(http_err) = azure_err.as_http_error() {