From 8d5d36ed12a2cfc0a0b9ccdc879ac0ca706a8e3c Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 4 Sep 2023 14:12:07 +0100 Subject: [PATCH] remote_storage: expose MAX_KEYS_PER_DELETE constant --- libs/remote_storage/src/lib.rs | 3 +++ libs/remote_storage/src/s3_bucket.rs | 7 +++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index a810f09787..8e8419cfd8 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -43,6 +43,9 @@ pub const DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT: usize = 100; /// pub const DEFAULT_MAX_KEYS_PER_LIST_RESPONSE: Option = None; +/// As defined in S3 docs +pub const MAX_KEYS_PER_DELETE: usize = 1000; + const REMOTE_STORAGE_PREFIX_SEPARATOR: char = '/'; /// Path on the remote storage, relative to some inner prefix. diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 9262f1e88f..acab953904 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -33,11 +33,10 @@ use tracing::debug; use super::StorageMetadata; use crate::{ - Download, DownloadError, RemotePath, RemoteStorage, S3Config, REMOTE_STORAGE_PREFIX_SEPARATOR, + Download, DownloadError, RemotePath, RemoteStorage, S3Config, MAX_KEYS_PER_DELETE, + REMOTE_STORAGE_PREFIX_SEPARATOR, }; -const MAX_DELETE_OBJECTS_REQUEST_SIZE: usize = 1000; - pub(super) mod metrics; use self::metrics::{AttemptOutcome, RequestKind}; @@ -500,7 +499,7 @@ impl RemoteStorage for S3Bucket { delete_objects.push(obj_id); } - for chunk in delete_objects.chunks(MAX_DELETE_OBJECTS_REQUEST_SIZE) { + for chunk in delete_objects.chunks(MAX_KEYS_PER_DELETE) { let started_at = start_measuring_requests(kind); let resp = self