ammending comit

Cargo.lock

@@ -37,6 +37,10 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "alek_ext"
+version = "0.1.0"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"

Cargo.toml

@@ -1,5 +1,6 @@
 [workspace]
 members = [
+    "alek_ext",
     "compute_tools",
     "control_plane",
     "pageserver",

alek_ext/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "alek_ext"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.71"
+remote_storage = { version = "0.1", path = "../libs/remote_storage/" }
+tokio = "1.28.2"
+toml_edit = "0.19.10"
+tracing = "0.1.37"
+
+[workspace]

alek_ext/alek.out

@@ -0,0 +1,5 @@
+this 
+Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+
+lol
+

alek_ext/src/local_PoC.rs

@@ -0,0 +1,65 @@
+/*
+ * The following code **works** for "downloading" a file stored at the fake bucket specified in
+ * `pageserver.toml` as :
+ * ```[remote_storage] 
+ * local_path = '../fakes3'```
+ *
+ * Next steps:
+ * 1. make it work with AWS
+ * 2. make it work for downloading multiple files, not just a single file
+ * 3. integrate it with compute_ctl
+ * 4. actually upload stuff to the bucket? so that it can be downloaded. Does this allow us to
+ *    modify `Dockerfile.computenode`? to delete the extension loading that is happening there?
+ * 5. How do the tenants upload extensions?
+ * 6. Maybe think about duplicating less stuff. 
+ * */
+
+use remote_storage::*;
+use std::path::Path;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use toml_edit;
+use anyhow;
+use tokio::io::AsyncReadExt;                                  
+
+async fn download_file() -> anyhow::Result<()> {
+    // read configurations from `pageserver.toml`
+    let cfg_file_path = Path::new("./../.neon/pageserver.toml");
+    let cfg_file_contents = std::fs::read_to_string(cfg_file_path).unwrap();
+    let toml = cfg_file_contents
+        .parse::<toml_edit::Document>()
+        .expect("Error parsing toml");
+    let remote_storage_data = toml.get("remote_storage")
+        .expect("field should be present");
+    let remote_storage_config = RemoteStorageConfig::from_toml(remote_storage_data)
+        .expect("error parsing toml")
+        .expect("error parsing toml");
+
+    // query S3 bucket
+    let remote_storage = GenericRemoteStorage::from_config(&remote_storage_config)?;
+    let from_path = "neon-dev-extensions/fuzzystrmatch.control";
+    let remote_from_path = RemotePath::new(Path::new(from_path))?;
+        
+    println!("im fine");
+    println!("{:?}",remote_storage_config);
+
+    let mut data = remote_storage.download(&remote_from_path).await.expect("data yay");
+    let mut write_data_buffer = Vec::new(); 
+
+    data.download_stream.read_to_end(&mut write_data_buffer).await?;
+
+    // write `data` to a file locally
+    let f = File::create("alek.out").expect("problem creating file");
+    let mut f = BufWriter::new(f);
+    f.write_all(&mut write_data_buffer).expect("error writing data");
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() {
+    match download_file().await {
+        Err(_)=>println!("Err"),
+        _ => println!("SUCEECESS")
+    }
+}

fakes3/neon-dev-extensions/fuzzystrmatch.control

@@ -0,0 +1,5 @@
+this 
+Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+
+lol
+

fakes3/neon-dev-extensions/pg_cron.control

@@ -0,0 +1,5 @@
+this 
+Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
+
+lol
+

libs/remote_storage/src/lib.rs

@@ -231,8 +231,10 @@ impl GenericRemoteStorage {
 
 impl GenericRemoteStorage {
     pub fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result<Self> {
+        println!("ALEK: is this fn fn even called");
         Ok(match &storage_config.storage {
             RemoteStorageKind::LocalFs(root) => {
+                println!("local{}", root.display());
                 info!("Using fs root '{}' as a remote storage", root.display());
                 Self::LocalFs(LocalFs::new(root.clone())?)
             }

libs/remote_storage/src/local_fs.rs

@@ -213,6 +213,7 @@ impl RemoteStorage for LocalFs {
 
     async fn download(&self, from: &RemotePath) -> Result<Download, DownloadError> {
         let target_path = from.with_base(&self.storage_root);
+        println!("{:?}",target_path);
         if file_exists(&target_path).map_err(DownloadError::BadInput)? {
             let source = io::BufReader::new(
                 fs::OpenOptions::new()