improvements to S3 bucket list_files

alek_ext/src/download_with_remote_api_v2.rs

@@ -40,8 +40,6 @@ async fn main() -> anyhow::Result<()> {
     let mut data = remote_storage.download(&remote_from_path).await.expect("data yay");
     let mut write_data_buffer = Vec::new(); 
     data.download_stream.read_to_end(&mut write_data_buffer).await?;
-
-    // write `data` to a file locally
     let f = File::create("alek.out").expect("problem creating file");
     let mut f = BufWriter::new(f);
     f.write_all(&mut write_data_buffer).expect("error writing data");

alek_ext/src/main.rs

@@ -1,9 +1,13 @@
 /* 
-**WIP**
  * This is a MWE of using our RemoteStorage API to call the aws stuff and download multiple files
- * TODO: s3_bucket some work; for example to make sure the pagination thing goes fine.
-macro_rules! alek { ($expression:expr) => { println!("{:?}", $expression); }; }
+ * 
+ * STATUS: 
+ * The s3bucket listing thing is, miracuously, working. 
+ * However, it is not really robust; please fix it to have 
+ * the features that the other functions have; pagination; permit; limit. 
+ * 
 */
+macro_rules! alek { ($expression:expr) => { println!("{:?}", $expression); }; }
 
 use remote_storage::*;
 use std::path::Path;
@@ -20,6 +24,7 @@ async fn main() -> anyhow::Result<()> {
     let subscriber = tracing_subscriber::FmtSubscriber::new();
     tracing::subscriber::set_global_default(subscriber)?;
 
+    // TODO: right now we are using the same config parameters as pageserver; but should we have our own configs?
     let cfg_file_path = Path::new("./../.neon/pageserver.toml");
     let cfg_file_contents = std::fs::read_to_string(cfg_file_path)
     .with_context(|| format!( "Failed to read pageserver config at '{}'", cfg_file_path.display()))?;
@@ -30,17 +35,17 @@ async fn main() -> anyhow::Result<()> {
         .context("field should be present")?;
     let remote_storage_config = RemoteStorageConfig::from_toml(remote_storage_data)?
         .context("error configuring remote storage")?;
-    info!("{:?}", remote_storage_config);
     let remote_storage = GenericRemoteStorage::from_config(&remote_storage_config)?;
 
     let folder = RemotePath::new(Path::new("public_extensions"))?;
     let from_paths = remote_storage.list_files(Some(&folder)).await?;
-
+    alek!(from_paths);
     for remote_from_path in from_paths {
         // TODO: where should we actually save the files to?
         if remote_from_path.extension() == Some("control") {
             let file_name = remote_from_path.object_name().expect("it must exist");
-            println!("{:?}",file_name);
+            info!("{:?}",file_name);
+            alek!(&remote_from_path);
             let mut download = remote_storage.download(&remote_from_path).await?;
             let mut write_data_buffer = Vec::new(); 
             download.download_stream.read_to_end(&mut write_data_buffer).await?;