diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 1d50a777f4..5b74308514 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -78,9 +78,6 @@ impl RemotePath { /// providing basic CRUD operations for storage files. #[async_trait::async_trait] pub trait RemoteStorage: Send + Sync + 'static { - /// Lists all items the storage has right now. - async fn list(&self) -> anyhow::Result>; - /// Lists all top level subdirectories for a given prefix /// Note: here we assume that if the prefix is passed it was obtained via remote_object_id /// which already takes into account any kind of global prefix (prefix_in_bucket for S3 or storage_root for LocalFS) diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs index f1289569ae..21a4156ad3 100644 --- a/libs/remote_storage/src/local_fs.rs +++ b/libs/remote_storage/src/local_fs.rs @@ -73,10 +73,8 @@ impl LocalFs { Ok(None) } } -} -#[async_trait::async_trait] -impl RemoteStorage for LocalFs { + #[cfg(test)] async fn list(&self) -> anyhow::Result> { Ok(get_all_files(&self.storage_root, true) .await? @@ -91,7 +89,10 @@ impl RemoteStorage for LocalFs { }) .collect()) } +} +#[async_trait::async_trait] +impl RemoteStorage for LocalFs { async fn list_prefixes( &self, prefix: Option<&RemotePath>, diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index d4eb7d9244..fdf3ae02d3 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -275,50 +275,6 @@ impl AsyncRead for RatelimitedAsyncRead { #[async_trait::async_trait] impl RemoteStorage for S3Bucket { - async fn list(&self) -> anyhow::Result> { - let mut document_keys = Vec::new(); - - let mut continuation_token = None; - loop { - let _guard = self - .concurrency_limiter - .acquire() - .await - .context("Concurrency limiter semaphore got closed during S3 list")?; - - metrics::inc_list_objects(); - - let fetch_response = self - .client - .list_objects_v2() - .bucket(self.bucket_name.clone()) - .set_prefix(self.prefix_in_bucket.clone()) - .delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()) - .set_continuation_token(continuation_token) - .set_max_keys(self.max_keys_per_list_response) - .send() - .await - .map_err(|e| { - metrics::inc_list_objects_fail(); - e - })?; - document_keys.extend( - fetch_response - .contents - .unwrap_or_default() - .into_iter() - .filter_map(|o| Some(self.s3_object_to_relative_path(o.key()?))), - ); - - match fetch_response.next_continuation_token { - Some(new_token) => continuation_token = Some(new_token), - None => break, - } - } - - Ok(document_keys) - } - /// See the doc for `RemoteStorage::list_prefixes` /// Note: it wont include empty "directories" async fn list_prefixes( diff --git a/libs/remote_storage/src/simulate_failures.rs b/libs/remote_storage/src/simulate_failures.rs index 643bb99dce..d1d062f8e7 100644 --- a/libs/remote_storage/src/simulate_failures.rs +++ b/libs/remote_storage/src/simulate_failures.rs @@ -20,7 +20,6 @@ pub struct UnreliableWrapper { /// Used to identify retries of different unique operation. #[derive(Debug, Hash, Eq, PartialEq)] enum RemoteOp { - List, ListPrefixes(Option), Upload(RemotePath), Download(RemotePath), @@ -75,12 +74,6 @@ impl UnreliableWrapper { #[async_trait::async_trait] impl RemoteStorage for UnreliableWrapper { - /// Lists all items the storage has right now. - async fn list(&self) -> anyhow::Result> { - self.attempt(RemoteOp::List)?; - self.inner.list().await - } - async fn list_prefixes( &self, prefix: Option<&RemotePath>,