This commit is contained in:
Alek Westover
2023-06-15 10:20:01 -04:00
parent 7465c644b9
commit 214ecacfc4
8 changed files with 60 additions and 27 deletions

View File

@@ -90,10 +90,10 @@ pub trait RemoteStorage: Send + Sync + 'static {
prefix: Option<&RemotePath>,
) -> Result<Vec<RemotePath>, DownloadError>;
/// Lists all files in a subdirectories
/// Lists all files in a folder
async fn list_files(
&self,
prefix: Option<&RemotePath>,
folder: Option<&RemotePath>,
) -> anyhow::Result<Vec<RemotePath>>;
/// Streams the local file contents into remote into the remote storage entry.

View File

@@ -336,7 +336,6 @@ impl RemoteStorage for S3Bucket {
&self,
folder: Option<&RemotePath>
) -> anyhow::Result<Vec<RemotePath>>{
// TODO: should we use DownloadError error type instead of anyhow::Error?
let folder_name = folder.map(|x|
String::from(x.object_name().expect("invalid folder name"))
);
@@ -348,7 +347,6 @@ impl RemoteStorage for S3Bucket {
.acquire()
.await
.context("Concurrency limiter semaphore got closed during S3 list_files")?;
metrics::inc_list_objects();
let response = self
.client
@@ -359,15 +357,10 @@ impl RemoteStorage for S3Bucket {
.set_max_keys(self.max_keys_per_list_response)
.send()
.await
.map_err(|e| {
metrics::inc_list_objects_fail();
e
})
.context("Failed to list files in S3 bucket")?;
for object in response.contents().unwrap_or_default() {
let object_path = object.key().unwrap();
println!("{:?}", object_path);
let remote_path = self.s3_object_to_relative_path(object_path);
all_files.push(remote_path);
}