more MWEs, stuff is starting to work

.gitignore

@@ -1,5 +1,6 @@
 /pg_install
 /target
+/alek_ext/target
 /tmp_check
 /tmp_check_cli
 __pycache__/

alek_ext/Cargo.lock

@@ -31,6 +31,9 @@ name = "alek_ext"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "aws-credential-types",
+ "aws-sdk-s3 0.28.0",
+ "aws-smithy-http",
  "remote_storage",
  "tokio",
  "toml_edit",
@@ -253,6 +256,39 @@ dependencies = [
  "url",
 ]
 
+[[package]]
+name = "aws-sdk-s3"
+version = "0.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fba197193cbb4bcb6aad8d99796b2291f36fa89562ded5d4501363055b0de89f"
+dependencies = [
+ "aws-credential-types",
+ "aws-endpoint",
+ "aws-http",
+ "aws-sig-auth",
+ "aws-sigv4",
+ "aws-smithy-async",
+ "aws-smithy-checksums",
+ "aws-smithy-client",
+ "aws-smithy-eventstream",
+ "aws-smithy-http",
+ "aws-smithy-http-tower",
+ "aws-smithy-json",
+ "aws-smithy-types",
+ "aws-smithy-xml",
+ "aws-types",
+ "bytes",
+ "http",
+ "http-body",
+ "once_cell",
+ "percent-encoding",
+ "regex",
+ "tokio-stream",
+ "tower",
+ "tracing",
+ "url",
+]
+
 [[package]]
 name = "aws-sdk-sts"
 version = "0.28.0"
@@ -1792,7 +1828,7 @@ dependencies = [
  "async-trait",
  "aws-config",
  "aws-credential-types",
- "aws-sdk-s3",
+ "aws-sdk-s3 0.27.0",
  "aws-smithy-http",
  "aws-types",
  "hyper",

alek_ext/Cargo.toml

@@ -7,6 +7,9 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1.0.71"
+aws-credential-types = "0.55.3"
+aws-sdk-s3 = "0.28.0"
+aws-smithy-http = "0.55.3"
 remote_storage = { version = "0.1", path = "../libs/remote_storage/" }
 tokio = "1.28.2"
 toml_edit = "0.19.10"

alek_ext/alek.out

@@ -1,5 +1,6 @@
-this 
-Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet.
-
-lol
-
+# fuzzystrmatch extension
+comment = 'determine similarities and distance between strings'
+default_version = '1.2'
+module_pathname = '$libdir/fuzzystrmatch'
+relocatable = true
+trusted = true

alek_ext/src/awsmwe_v1.rs

@@ -0,0 +1,29 @@
+use aws_sdk_s3::{self, config::Region, Error};
+use aws_config::{self, meta::region::RegionProviderChain};
+
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    let region_provider = RegionProviderChain::first_try(Region::new("eu-central-1"))
+        .or_default_provider()
+        .or_else(Region::new("eu-central-1"));
+
+    let shared_config = aws_config::from_env().region(region_provider).load().await;
+    let client = aws_sdk_s3::Client::new(&shared_config);
+
+    let bucket_name = "neon-dev-extensions";
+    let object_key = "fuzzystrmatch.control";
+    let response = client
+        .get_object()
+        .bucket(bucket_name)
+        .key(object_key)
+        .send()
+        .await?;
+
+    let stuff = response.body;
+    let data = stuff.collect().await.expect("error reading data").to_vec();
+    println!("data: {:?}", std::str::from_utf8(&data));
+
+    Ok(())
+}
+

alek_ext/src/complicated.rs

@@ -0,0 +1,77 @@
+/*
+ * The following code attempts to actually download a file from the S3 bucket specified in
+ * `pageserver.toml` as :
+ * ```[remote_storage] 
+ * bucket_name = 'neon-dev-extensions'
+ * bucket_region = 'eu-central-1'
+ *
+ * Note: must run `export AWS_PROFILE=PowerUserAccess-11111111111` for your SSO credentials
+ * to get loaded; alternatively go get other credentials. But SSO is better.
+ *
+ * Next steps:
+ * 1. **make it work with AWS** (this code hopefully!)
+ * 2. make it work for downloading multiple files, not just a single file
+ * 3. integrate it with compute_ctl
+ * 4. actually upload stuff to the bucket? so that it can be downloaded. Does this allow us to
+ *    modify `Dockerfile.computenode`? to delete the extension loading that is happening there?
+ * 5. How do the tenants upload extensions?
+ * 6. Maybe think about duplicating less stuff. 
+ * */
+
+use remote_storage::*;
+use std::path::Path;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use toml_edit;
+use anyhow;
+use tokio::io::AsyncReadExt;                                  
+use remote_storage::GenericRemoteStorage::AwsS3;
+
+async fn download_file() -> anyhow::Result<()> {
+    // read configurations from `pageserver.toml`
+    let cfg_file_path = Path::new("./../.neon/pageserver.toml");
+    let cfg_file_contents = std::fs::read_to_string(cfg_file_path).unwrap();
+    let toml = cfg_file_contents
+        .parse::<toml_edit::Document>()
+        .expect("Error parsing toml");
+    let remote_storage_data = toml.get("remote_storage")
+        .expect("field should be present");
+    let remote_storage_config = RemoteStorageConfig::from_toml(remote_storage_data)
+        .expect("error parsing toml")
+        .expect("error parsing toml");
+
+    println!("CONFIG LGTM!!!\n {:?}", remote_storage_config);
+
+    // query S3 bucket
+    let remote_storage = GenericRemoteStorage::from_config(&remote_storage_config)?;
+    let from_path = "neon-dev-extensions/fuzzystrmatch.control";
+    let remote_from_path = RemotePath::new(Path::new(from_path))?;
+        
+    if let AwsS3(printablebucket) = &remote_storage {
+        println!("S3Bucket looks fine, AFAICT{:?}", printablebucket);
+    }
+    println!("{:?}"&remote_from_path);
+
+    let mut data = remote_storage.download(&remote_from_path).await;
+
+    /*
+    let mut write_data_buffer = Vec::new(); 
+
+    data.download_stream.read_to_end(&mut write_data_buffer).await?;
+
+    // write `data` to a file locally
+    let f = File::create("alek.out").expect("problem creating file");
+    let mut f = BufWriter::new(f);
+    f.write_all(&mut write_data_buffer).expect("error writing data");
+    */
+
+    Ok(())
+}
+
+#[tokio::main]
+async fn main() {
+    match download_file().await {
+        Err(_)=>println!("Err"),
+        _ => println!("SUCEECESS")
+    }
+}

alek_ext/src/main.rs

@@ -0,0 +1,66 @@
+use remote_storage::*;
+use std::path::Path;
+use std::fs::File;
+use std::io::{BufWriter, Write};
+use toml_edit;
+use anyhow;
+use tokio::io::AsyncReadExt;                                  
+
+// let region_provider = RegionProviderChain::first_try(Region::new("eu-central-1"))
+//     .or_default_provider()
+//     .or_else(Region::new("eu-central-1"));
+
+// let shared_config = aws_config::from_env().region(region_provider).load().await;
+// let client = aws_sdk_s3::Client::new(&shared_config);
+
+// let bucket_name = "neon-dev-extensions";
+// let object_key = "fuzzystrmatch.control";
+// let response = client
+//     .get_object()
+//     .bucket(bucket_name)
+//     .key(object_key)
+//     .send()
+//     .await?;
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let from_path = "fuzzystrmatch.control";
+    let remote_from_path = RemotePath::new(Path::new(from_path))?;
+    println!("{:?}", remote_from_path.clone());
+
+    // read configurations from `pageserver.toml`
+    let cfg_file_path = Path::new("./../.neon/pageserver.toml");
+    let cfg_file_contents = std::fs::read_to_string(cfg_file_path).unwrap();
+    let toml = cfg_file_contents
+        .parse::<toml_edit::Document>()
+        .expect("Error parsing toml");
+    let remote_storage_data = toml.get("remote_storage")
+        .expect("field should be present");
+    let remote_storage_config = RemoteStorageConfig::from_toml(remote_storage_data)
+        .expect("error parsing toml")
+        .expect("error parsing toml");
+
+    // query S3 bucket
+    let remote_storage = GenericRemoteStorage::from_config(&remote_storage_config)?;
+    let from_path = "fuzzystrmatch.control";
+    let remote_from_path = RemotePath::new(Path::new(from_path))?;
+        
+    println!("{:?}", remote_from_path.clone());
+    // if let GenericRemoteStorage::AwsS3(mybucket) = remote_storage {
+    //     println!("{:?}",mybucket.relative_path_to_s3_object(&remote_from_path));
+    // }
+    let mut data = remote_storage.download(&remote_from_path).await.expect("data yay");
+    let mut write_data_buffer = Vec::new(); 
+    data.download_stream.read_to_end(&mut write_data_buffer).await?;
+
+    // write `data` to a file locally
+    let f = File::create("alek.out").expect("problem creating file");
+    let mut f = BufWriter::new(f);
+    f.write_all(&mut write_data_buffer).expect("error writing data");
+
+    // let stuff = response.body;
+    // let data = stuff.collect().await.expect("error reading data").to_vec();
+    // println!("data: {:?}", std::str::from_utf8(&data));
+
+    Ok(())
+}

libs/remote_storage/src/lib.rs

@@ -231,10 +231,8 @@ impl GenericRemoteStorage {
 
 impl GenericRemoteStorage {
     pub fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result<Self> {
-        println!("ALEK: is this fn fn even called");
         Ok(match &storage_config.storage {
             RemoteStorageKind::LocalFs(root) => {
-                println!("local{}", root.display());
                 info!("Using fs root '{}' as a remote storage", root.display());
                 Self::LocalFs(LocalFs::new(root.clone())?)
             }