diff --git a/ALEK_LIST_FILES.txt b/ALEK_LIST_FILES.txt index 3ef734e640..698472fafd 100644 --- a/ALEK_LIST_FILES.txt +++ b/ALEK_LIST_FILES.txt @@ -1 +1 @@ -[RemotePath("v15/share/extension/test_ext.control")] \ No newline at end of file +[RemotePath("v14/share/postgresql/extension/test_ext.control")] diff --git a/ALEK_SHAREDIR.txt b/ALEK_SHAREDIR.txt new file mode 100644 index 0000000000..9735be8b4a --- /dev/null +++ b/ALEK_SHAREDIR.txt @@ -0,0 +1 @@ +"v14/share/postgresql/extension" \ No newline at end of file diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index c68e361ee3..9f2caa06cd 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -70,9 +70,8 @@ fn main() -> Result<()> { .expect("remote-extension-config is required"); let remote_storage = init_remote_storage(remote_ext_config)?; - // TODO: can we give remote_storage a static lifetime, so that we don't have to copy it? - let copy_remote_storage = remote_storage.clone(); let rt = Runtime::new().unwrap(); + let copy_remote_storage = remote_storage.clone(); rt.block_on(async move { download_extension(©_remote_storage, ExtensionType::Shared, &pgbin) .await diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 24455c0f82..b749f6e927 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -9,8 +9,8 @@ use std::str; use tokio::io::AsyncReadExt; use tracing::info; -fn get_pg_config(argument: &str, pgbin: &str) -> String { - let mut pgconfig = String::from(pgbin.strip_suffix("postgres").unwrap()); +fn get_pg_config(argument: &str, pgbin: &str) -> (String, String) { + let mut pgconfig = String::from(pgbin.strip_suffix("postgres").expect("pg_config error")); pgconfig.push_str("pg_config"); let config_output = std::process::Command::new(pgconfig) @@ -18,21 +18,35 @@ fn get_pg_config(argument: &str, pgbin: &str) -> String { .output() .expect("pg_config must be installed"); assert!(config_output.status.success()); - let stdout = std::str::from_utf8(&config_output.stdout).expect("error obtaining pg_config"); - stdout.trim().to_string() + let local_path = std::str::from_utf8(&config_output.stdout) + .expect("error obtaining pg_config") + .trim() + .to_string(); + + let mut rm_prefix: String = std::env::current_dir() + .expect("pg_config error") + .to_str() + .expect("pg_config error") + .into(); + rm_prefix.push_str("/pg_install/"); + let remote_path = local_path + .strip_prefix(&rm_prefix) + .expect("pg_config error") + .trim() + .to_string(); + + (local_path, remote_path) } async fn download_helper( remote_storage: &GenericRemoteStorage, remote_from_path: &RemotePath, - to_path: &str, + download_to_dir: &str, ) -> anyhow::Result<()> { + std::fs::write("ALEK_DOWNLOAD.txt", format!("{:?}", download_to_dir))?; let file_name = remote_from_path.object_name().expect("it must exist"); info!("Downloading {:?}", file_name); - info!( - "To location {:?} (actually just downloading it with it's remote name for now at least)", - to_path - ); + info!("To location {:?}", download_to_dir); let mut download = remote_storage.download(&remote_from_path).await?; let mut write_data_buffer = Vec::new(); download @@ -58,21 +72,25 @@ pub async fn download_extension( let from_paths = remote_storage.list_files(None).await?; std::fs::write("ALEK_LIST_FILES.txt", format!("{:?}", from_paths))?; - // TODO: probably should be using the pgbin argv somehow to compute sharedir..., - // right now it is getting my global pg_config, which is wrong - let sharedir = get_pg_config("--sharedir", pgbin); - let sharedir = format!("{}/extension", sharedir); - let libdir = get_pg_config("--libdir", pgbin); + let (mut local_sharedir, mut remote_sharedir) = get_pg_config("--sharedir", pgbin); + local_sharedir.push_str("/extension"); + remote_sharedir.push_str("/extension"); + std::fs::write("ALEK_SHAREDIR.txt", format!("{:?}", remote_sharedir))?; + let (local_libdir, _) = get_pg_config("--libdir", pgbin); match ext_type { ExtensionType::Shared => { // 1. Download control files from s3-bucket/public/*.control to SHAREDIR/extension // We can do this step even before we have spec, // because public extensions are common for all projects. - let folder = RemotePath::new(Path::new("public_extensions"))?; + let folder = RemotePath::new(Path::new(&remote_sharedir))?; let from_paths = remote_storage.list_files(Some(&folder)).await?; + std::fs::write( + "ALEK_QUEUE_DOWNLOAD.txt", + format!("{:?}", from_paths.clone()), + )?; for remote_from_path in from_paths { if remote_from_path.extension() == Some("control") { - download_helper(&remote_storage, &remote_from_path, &sharedir).await?; + download_helper(&remote_storage, &remote_from_path, &local_sharedir).await?; } } } @@ -83,7 +101,7 @@ pub async fn download_extension( let from_paths = remote_storage.list_files(Some(&folder)).await?; for remote_from_path in from_paths { if remote_from_path.extension() == Some("control") { - download_helper(&remote_storage, &remote_from_path, &sharedir).await?; + download_helper(&remote_storage, &remote_from_path, &local_sharedir).await?; } } } @@ -92,7 +110,7 @@ pub async fn download_extension( // Download preload_shared_libraries from s3-bucket/public/[library-name].control into LIBDIR/ let from_path = format!("neon-dev-extensions/public/{library_name}.control"); let remote_from_path = RemotePath::new(Path::new(&from_path))?; - download_helper(&remote_storage, &remote_from_path, &libdir).await?; + download_helper(&remote_storage, &remote_from_path, &local_libdir).await?; } } Ok(()) diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py index 6429888121..3154abd6b2 100644 --- a/test_runner/regress/test_download_extensions.py +++ b/test_runner/regress/test_download_extensions.py @@ -6,8 +6,6 @@ from fixtures.neon_fixtures import ( ) import json -TEST_EXT_PATH = "v15/share/extension/test_ext.control" - def test_file_download(neon_env_builder: NeonEnvBuilder): """ @@ -19,7 +17,7 @@ def test_file_download(neon_env_builder: NeonEnvBuilder): Right now we are downloading the file in python However, we have all the argument passing set up so that when an endpoint starts it knows about the bucket and can list_files in the bucket. This is written to ALEK_LIST_FILES.txt - A good next step is to get rust to downlaod the public_extensions control files to the correct place + A good next step is to get rust to download the public_extensions control files to the correct place """ neon_env_builder.enable_remote_storage( remote_storage_kind=RemoteStorageKind.MOCK_S3, @@ -28,6 +26,8 @@ def test_file_download(neon_env_builder: NeonEnvBuilder): neon_env_builder.num_safekeepers = 3 env = neon_env_builder.init_start() + TEST_EXT_PATH = "v14/share/postgresql/extension/test_ext.control" + # 4. Upload test_ext.control file to the bucket # In the non-mock version this is done by CI/CD with open("test_ext.control", "rb") as data: @@ -41,10 +41,9 @@ def test_file_download(neon_env_builder: NeonEnvBuilder): Bucket=env.ext_remote_storage.bucket_name, Key=TEST_EXT_PATH ) response = resp["Body"] - for pgres_version in ("v15", "v14"): - fname = f"pg_install/{pgres_version}/share/postgresql/extension/test_ext.control" - with open(fname, "wb") as f: - f.write(response.read()) + fname = f"pg_install/{TEST_EXT_PATH}" + with open(fname, "wb") as f: + f.write(response.read()) tenant, _ = env.neon_cli.create_tenant() env.neon_cli.create_timeline("test_file_download", tenant_id=tenant)