diff --git a/.gitignore b/.gitignore index f1afdee599..e3960f6afc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /pg_install /target +/alek_ext/target /tmp_check /tmp_check_cli __pycache__/ diff --git a/alek_ext/Cargo.lock b/alek_ext/Cargo.lock index 977ab32e7e..2a011e82c7 100644 --- a/alek_ext/Cargo.lock +++ b/alek_ext/Cargo.lock @@ -31,6 +31,9 @@ name = "alek_ext" version = "0.1.0" dependencies = [ "anyhow", + "aws-credential-types", + "aws-sdk-s3 0.28.0", + "aws-smithy-http", "remote_storage", "tokio", "toml_edit", @@ -253,6 +256,39 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-s3" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba197193cbb4bcb6aad8d99796b2291f36fa89562ded5d4501363055b0de89f" +dependencies = [ + "aws-credential-types", + "aws-endpoint", + "aws-http", + "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tokio-stream", + "tower", + "tracing", + "url", +] + [[package]] name = "aws-sdk-sts" version = "0.28.0" @@ -1792,7 +1828,7 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", - "aws-sdk-s3", + "aws-sdk-s3 0.27.0", "aws-smithy-http", "aws-types", "hyper", diff --git a/alek_ext/Cargo.toml b/alek_ext/Cargo.toml index 911c7f185f..5ac9811953 100644 --- a/alek_ext/Cargo.toml +++ b/alek_ext/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" [dependencies] anyhow = "1.0.71" +aws-credential-types = "0.55.3" +aws-sdk-s3 = "0.28.0" +aws-smithy-http = "0.55.3" remote_storage = { version = "0.1", path = "../libs/remote_storage/" } tokio = "1.28.2" toml_edit = "0.19.10" diff --git a/alek_ext/alek.out b/alek_ext/alek.out index fd6f98abc5..8b6e9fd993 100644 --- a/alek_ext/alek.out +++ b/alek_ext/alek.out @@ -1,5 +1,6 @@ -this -Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. - -lol - +# fuzzystrmatch extension +comment = 'determine similarities and distance between strings' +default_version = '1.2' +module_pathname = '$libdir/fuzzystrmatch' +relocatable = true +trusted = true diff --git a/alek_ext/src/awsmwe_v1.rs b/alek_ext/src/awsmwe_v1.rs new file mode 100644 index 0000000000..60e3a1c656 --- /dev/null +++ b/alek_ext/src/awsmwe_v1.rs @@ -0,0 +1,29 @@ +use aws_sdk_s3::{self, config::Region, Error}; +use aws_config::{self, meta::region::RegionProviderChain}; + + +#[tokio::main] +async fn main() -> Result<(), Error> { + let region_provider = RegionProviderChain::first_try(Region::new("eu-central-1")) + .or_default_provider() + .or_else(Region::new("eu-central-1")); + + let shared_config = aws_config::from_env().region(region_provider).load().await; + let client = aws_sdk_s3::Client::new(&shared_config); + + let bucket_name = "neon-dev-extensions"; + let object_key = "fuzzystrmatch.control"; + let response = client + .get_object() + .bucket(bucket_name) + .key(object_key) + .send() + .await?; + + let stuff = response.body; + let data = stuff.collect().await.expect("error reading data").to_vec(); + println!("data: {:?}", std::str::from_utf8(&data)); + + Ok(()) +} + diff --git a/alek_ext/src/complicated.rs b/alek_ext/src/complicated.rs new file mode 100644 index 0000000000..8fb78fe688 --- /dev/null +++ b/alek_ext/src/complicated.rs @@ -0,0 +1,77 @@ +/* + * The following code attempts to actually download a file from the S3 bucket specified in + * `pageserver.toml` as : + * ```[remote_storage] + * bucket_name = 'neon-dev-extensions' + * bucket_region = 'eu-central-1' + * + * Note: must run `export AWS_PROFILE=PowerUserAccess-11111111111` for your SSO credentials + * to get loaded; alternatively go get other credentials. But SSO is better. + * + * Next steps: + * 1. **make it work with AWS** (this code hopefully!) + * 2. make it work for downloading multiple files, not just a single file + * 3. integrate it with compute_ctl + * 4. actually upload stuff to the bucket? so that it can be downloaded. Does this allow us to + * modify `Dockerfile.computenode`? to delete the extension loading that is happening there? + * 5. How do the tenants upload extensions? + * 6. Maybe think about duplicating less stuff. + * */ + +use remote_storage::*; +use std::path::Path; +use std::fs::File; +use std::io::{BufWriter, Write}; +use toml_edit; +use anyhow; +use tokio::io::AsyncReadExt; +use remote_storage::GenericRemoteStorage::AwsS3; + +async fn download_file() -> anyhow::Result<()> { + // read configurations from `pageserver.toml` + let cfg_file_path = Path::new("./../.neon/pageserver.toml"); + let cfg_file_contents = std::fs::read_to_string(cfg_file_path).unwrap(); + let toml = cfg_file_contents + .parse::() + .expect("Error parsing toml"); + let remote_storage_data = toml.get("remote_storage") + .expect("field should be present"); + let remote_storage_config = RemoteStorageConfig::from_toml(remote_storage_data) + .expect("error parsing toml") + .expect("error parsing toml"); + + println!("CONFIG LGTM!!!\n {:?}", remote_storage_config); + + // query S3 bucket + let remote_storage = GenericRemoteStorage::from_config(&remote_storage_config)?; + let from_path = "neon-dev-extensions/fuzzystrmatch.control"; + let remote_from_path = RemotePath::new(Path::new(from_path))?; + + if let AwsS3(printablebucket) = &remote_storage { + println!("S3Bucket looks fine, AFAICT{:?}", printablebucket); + } + println!("{:?}"&remote_from_path); + + let mut data = remote_storage.download(&remote_from_path).await; + + /* + let mut write_data_buffer = Vec::new(); + + data.download_stream.read_to_end(&mut write_data_buffer).await?; + + // write `data` to a file locally + let f = File::create("alek.out").expect("problem creating file"); + let mut f = BufWriter::new(f); + f.write_all(&mut write_data_buffer).expect("error writing data"); + */ + + Ok(()) +} + +#[tokio::main] +async fn main() { + match download_file().await { + Err(_)=>println!("Err"), + _ => println!("SUCEECESS") + } +} diff --git a/alek_ext/src/local_PoC.rs b/alek_ext/src/localfiledownload_v0.rs similarity index 100% rename from alek_ext/src/local_PoC.rs rename to alek_ext/src/localfiledownload_v0.rs diff --git a/alek_ext/src/main.rs b/alek_ext/src/main.rs new file mode 100644 index 0000000000..66407c68ce --- /dev/null +++ b/alek_ext/src/main.rs @@ -0,0 +1,66 @@ +use remote_storage::*; +use std::path::Path; +use std::fs::File; +use std::io::{BufWriter, Write}; +use toml_edit; +use anyhow; +use tokio::io::AsyncReadExt; + +// let region_provider = RegionProviderChain::first_try(Region::new("eu-central-1")) +// .or_default_provider() +// .or_else(Region::new("eu-central-1")); + +// let shared_config = aws_config::from_env().region(region_provider).load().await; +// let client = aws_sdk_s3::Client::new(&shared_config); + +// let bucket_name = "neon-dev-extensions"; +// let object_key = "fuzzystrmatch.control"; +// let response = client +// .get_object() +// .bucket(bucket_name) +// .key(object_key) +// .send() +// .await?; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let from_path = "fuzzystrmatch.control"; + let remote_from_path = RemotePath::new(Path::new(from_path))?; + println!("{:?}", remote_from_path.clone()); + + // read configurations from `pageserver.toml` + let cfg_file_path = Path::new("./../.neon/pageserver.toml"); + let cfg_file_contents = std::fs::read_to_string(cfg_file_path).unwrap(); + let toml = cfg_file_contents + .parse::() + .expect("Error parsing toml"); + let remote_storage_data = toml.get("remote_storage") + .expect("field should be present"); + let remote_storage_config = RemoteStorageConfig::from_toml(remote_storage_data) + .expect("error parsing toml") + .expect("error parsing toml"); + + // query S3 bucket + let remote_storage = GenericRemoteStorage::from_config(&remote_storage_config)?; + let from_path = "fuzzystrmatch.control"; + let remote_from_path = RemotePath::new(Path::new(from_path))?; + + println!("{:?}", remote_from_path.clone()); + // if let GenericRemoteStorage::AwsS3(mybucket) = remote_storage { + // println!("{:?}",mybucket.relative_path_to_s3_object(&remote_from_path)); + // } + let mut data = remote_storage.download(&remote_from_path).await.expect("data yay"); + let mut write_data_buffer = Vec::new(); + data.download_stream.read_to_end(&mut write_data_buffer).await?; + + // write `data` to a file locally + let f = File::create("alek.out").expect("problem creating file"); + let mut f = BufWriter::new(f); + f.write_all(&mut write_data_buffer).expect("error writing data"); + + // let stuff = response.body; + // let data = stuff.collect().await.expect("error reading data").to_vec(); + // println!("data: {:?}", std::str::from_utf8(&data)); + + Ok(()) +} diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 3ee84c5ab4..ec52448025 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -231,10 +231,8 @@ impl GenericRemoteStorage { impl GenericRemoteStorage { pub fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result { - println!("ALEK: is this fn fn even called"); Ok(match &storage_config.storage { RemoteStorageKind::LocalFs(root) => { - println!("local{}", root.display()); info!("Using fs root '{}' as a remote storage", root.display()); Self::LocalFs(LocalFs::new(root.clone())?) }