diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 34c7463bb4..e6717007c9 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -77,15 +77,20 @@ fn main() -> Result<()> { Value::String(x) => x, _ => panic!("oops"), }; + let remote_ext_endpoint = match &remote_ext_config["endpoint"] { + Value::String(x) => x, + _ => panic!("oops"), + }; warn!("you certainly must build changes if you want rust changes to be built"); std::fs::write("alek/yay", remote_ext_bucket.clone())?; - let mut rt = Runtime::new().unwrap(); + let rt = Runtime::new().unwrap(); rt.block_on(async move { compute_tools::extension_server::download_file( "test_ext.control", remote_ext_bucket.into(), remote_ext_region.into(), + remote_ext_endpoint.into(), ) .await .expect("download should work"); @@ -210,6 +215,7 @@ fn main() -> Result<()> { state_changed: Condvar::new(), remote_ext_bucket: remote_ext_bucket.clone(), // TODO ALEK: pass all the args! remote_ext_region: remote_ext_region.clone(), + remote_ext_endpoint: remote_ext_endpoint.clone(), }; let compute = Arc::new(compute_node); diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index a0b61a986f..b48e07431e 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -49,6 +49,7 @@ pub struct ComputeNode { // TODO: alek pass all args here pub remote_ext_bucket: String, pub remote_ext_region: String, + pub remote_ext_endpoint: String, } #[derive(Clone, Debug)] diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 905126aab6..3ea2e87a93 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -13,6 +13,7 @@ pub async fn download_file( filename: &str, remote_ext_bucket: String, remote_ext_region: String, + remote_ext_endpoint: String, ) -> anyhow::Result<()> { // probably should be using the pgbin argv somehow to compute sharedir... let sharedir = get_pg_config("--sharedir"); @@ -21,7 +22,7 @@ pub async fn download_file( println!("requested file {}", filename); // TODO: download the extensions! - let s3_config = create_s3_config(remote_ext_bucket, remote_ext_region); + let s3_config = create_s3_config(remote_ext_bucket, remote_ext_region, remote_ext_endpoint); download_extension(&s3_config, ExtensionType::Shared).await?; // This is filler code @@ -46,7 +47,7 @@ fn get_pg_config(argument: &str) -> String { stdout.trim().to_string() } -async fn download_helper( +async fn _download_helper( remote_storage: &GenericRemoteStorage, remote_from_path: &RemotePath, to_path: &str, @@ -83,11 +84,11 @@ pass config to compute_ctl */ pub async fn download_extension( config: &RemoteStorageConfig, - ext_type: ExtensionType, + _ext_type: ExtensionType, ) -> anyhow::Result<()> { let sharedir = get_pg_config("--sharedir"); - let sharedir = format!("{}/extension", sharedir); - let libdir = get_pg_config("--libdir"); + let _sharedir = format!("{}/extension", sharedir); + // let libdir = get_pg_config("--libdir"); let remote_storage = GenericRemoteStorage::from_config(config)?; std::fs::write("alek/proof", "proof")?; @@ -97,11 +98,17 @@ pub async fn download_extension( my_bucket.bucket_name, my_bucket.prefix_in_bucket ); std::fs::write("alek/storagedetails", storage_details)?; + dbg!(my_bucket.client.clone()); + for _ in 0..100 { + dbg!("++++++++++"); + } } // // this is just for testing doing a testing thing let folder = RemotePath::new(Path::new("public_extensions"))?; let from_paths = remote_storage.list_files(Some(&folder)).await?; + dbg!(from_paths); + dbg!("FROM PATHS!!!!!1"); std::fs::write("alek/antiproof", "antiproof")?; // let some_path = from_paths[0] // .object_name() @@ -150,12 +157,17 @@ pub async fn download_extension( pub fn create_s3_config( remote_ext_bucket: String, remote_ext_region: String, + remote_ext_endpoint: String, ) -> RemoteStorageConfig { + dbg!("create_s3_config"); + dbg!(&remote_ext_bucket); + dbg!(&remote_ext_region); + dbg!(&remote_ext_endpoint); let config = S3Config { bucket_name: remote_ext_bucket, bucket_region: remote_ext_region, prefix_in_bucket: None, - endpoint: None, + endpoint: Some(remote_ext_endpoint), concurrency_limit: NonZeroUsize::new(100).expect("100 != 0"), max_keys_per_list_response: None, }; diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index f8aab17d5d..2e0fa2878a 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -140,6 +140,7 @@ async fn routes(req: Request, compute: &Arc) -> Response, max_keys_per_list_response: Option, diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py index 0163e3abbd..17d8f93714 100644 --- a/test_runner/regress/test_download_extensions.py +++ b/test_runner/regress/test_download_extensions.py @@ -71,7 +71,8 @@ def test_file_download(neon_env_builder: NeonEnvBuilder): remote_ext_config = json.dumps( { "bucket": neon_env_builder.remote_storage.bucket_name, - "region": "eu-central-1", + "region": "us-east-1", + "endpoint": neon_env_builder.remote_storage.endpoint, } )