From 30815582a7fa76d42737ba3c4ab73adddbef4f9b Mon Sep 17 00:00:00 2001
From: Alek Westover <alek.westover@gmail.com>
Date: Wed, 14 Jun 2023 08:48:06 -0400
Subject: [PATCH] improvements to S3 bucket list_files

---
 .../src/{real_main.rs => backup_something.rs}   |  0
 alek_ext/src/download_with_remote_api_v2.rs     |  2 --
 alek_ext/src/main.rs                            | 17 +++++++++++------
 3 files changed, 11 insertions(+), 8 deletions(-)
 rename alek_ext/src/{real_main.rs => backup_something.rs} (100%)

diff --git a/alek_ext/src/real_main.rs b/alek_ext/src/backup_something.rs
similarity index 100%
rename from alek_ext/src/real_main.rs
rename to alek_ext/src/backup_something.rs
diff --git a/alek_ext/src/download_with_remote_api_v2.rs b/alek_ext/src/download_with_remote_api_v2.rs
index 37dca9c278..5dc0e0e36e 100644
--- a/alek_ext/src/download_with_remote_api_v2.rs
+++ b/alek_ext/src/download_with_remote_api_v2.rs
@@ -40,8 +40,6 @@ async fn main() -> anyhow::Result<()> {
     let mut data = remote_storage.download(&remote_from_path).await.expect("data yay");
     let mut write_data_buffer = Vec::new(); 
     data.download_stream.read_to_end(&mut write_data_buffer).await?;
-
-    // write `data` to a file locally
     let f = File::create("alek.out").expect("problem creating file");
     let mut f = BufWriter::new(f);
     f.write_all(&mut write_data_buffer).expect("error writing data");
diff --git a/alek_ext/src/main.rs b/alek_ext/src/main.rs
index cae774b0d9..096c9a024d 100644
--- a/alek_ext/src/main.rs
+++ b/alek_ext/src/main.rs
@@ -1,9 +1,13 @@
 /* 
-**WIP**
  * This is a MWE of using our RemoteStorage API to call the aws stuff and download multiple files
- * TODO: s3_bucket some work; for example to make sure the pagination thing goes fine.
-macro_rules! alek { ($expression:expr) => { println!("{:?}", $expression); }; }
+ * 
+ * STATUS: 
+ * The s3bucket listing thing is, miracuously, working. 
+ * However, it is not really robust; please fix it to have 
+ * the features that the other functions have; pagination; permit; limit. 
+ * 
 */
+macro_rules! alek { ($expression:expr) => { println!("{:?}", $expression); }; }
 
 use remote_storage::*;
 use std::path::Path;
@@ -20,6 +24,7 @@ async fn main() -> anyhow::Result<()> {
     let subscriber = tracing_subscriber::FmtSubscriber::new();
     tracing::subscriber::set_global_default(subscriber)?;
 
+    // TODO: right now we are using the same config parameters as pageserver; but should we have our own configs?
     let cfg_file_path = Path::new("./../.neon/pageserver.toml");
     let cfg_file_contents = std::fs::read_to_string(cfg_file_path)
     .with_context(|| format!( "Failed to read pageserver config at '{}'", cfg_file_path.display()))?;
@@ -30,17 +35,17 @@ async fn main() -> anyhow::Result<()> {
         .context("field should be present")?;
     let remote_storage_config = RemoteStorageConfig::from_toml(remote_storage_data)?
         .context("error configuring remote storage")?;
-    info!("{:?}", remote_storage_config);
     let remote_storage = GenericRemoteStorage::from_config(&remote_storage_config)?;
 
     let folder = RemotePath::new(Path::new("public_extensions"))?;
     let from_paths = remote_storage.list_files(Some(&folder)).await?;
-
+    alek!(from_paths);
     for remote_from_path in from_paths {
         // TODO: where should we actually save the files to?
         if remote_from_path.extension() == Some("control") {
             let file_name = remote_from_path.object_name().expect("it must exist");
-            println!("{:?}",file_name);
+            info!("{:?}",file_name);
+            alek!(&remote_from_path);
             let mut download = remote_storage.download(&remote_from_path).await?;
             let mut write_data_buffer = Vec::new(); 
             download.download_stream.read_to_end(&mut write_data_buffer).await?;