diff --git a/Cargo.lock b/Cargo.lock index d390df94e0..697f8729e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -924,6 +924,7 @@ dependencies = [ "opentelemetry", "postgres", "regex", + "remote_storage", "reqwest", "serde", "serde_json", diff --git a/alekbuilderror b/alekbuilderror new file mode 100644 index 0000000000..889a1e216a --- /dev/null +++ b/alekbuilderror @@ -0,0 +1,68 @@ + Compiling compute_tools v0.1.0 (/home/alek/Desktop/neon0/compute_tools) +error[E0599]: no method named `with_context` found for enum `Result` in the current scope + --> compute_tools/src/pg_extensions.rs:71:10 + | +71 | .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?; + | ^^^^^^^^^^^^ method not found in `Result` + | + = help: items from traits can only be used if the trait is in scope +help: the following traits are implemented but not in scope; perhaps add a `use` for one of them: + | +2 | use anyhow::Context; + | +2 | use opentelemetry_api::trace::context::FutureExt; + | + +error[E0425]: cannot find function `initialize_config` in this scope + --> compute_tools/src/pg_extensions.rs:75:22 + | +75 | let conf = match initialize_config(&cfg_file_path, arg_matches, &workdir)? { + | ^^^^^^^^^^^^^^^^^ not found in this scope + +error[E0308]: mismatched types + --> compute_tools/src/pg_extensions.rs:79:23 + | +79 | return Ok(()); + | -- ^^ expected struct `RemoteStorageConfig`, found `()` + | | + | arguments to this enum variant are incorrect + | +help: the type constructed contains `()` due to the type of the argument passed + --> compute_tools/src/pg_extensions.rs:79:20 + | +79 | return Ok(()); + | ^^^--^ + | | + | this argument influences the type of `Ok` +note: tuple variant defined here + --> /home/alek/.rustup/toolchains/1.68.2-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:507:5 + | +507 | Ok(#[stable(feature = "rust1", since = "1.0.0")] T), + | ^^ + +error[E0308]: mismatched types + --> compute_tools/src/pg_extensions.rs:86:19 + | +86 | return Ok(None); + | -- ^^^^ expected struct `RemoteStorageConfig`, found enum `Option` + | | + | arguments to this enum variant are incorrect + | + = note: expected struct `remote_storage::RemoteStorageConfig` + found enum `Option<_>` +help: the type constructed contains `Option<_>` due to the type of the argument passed + --> compute_tools/src/pg_extensions.rs:86:16 + | +86 | return Ok(None); + | ^^^----^ + | | + | this argument influences the type of `Ok` +note: tuple variant defined here + --> /home/alek/.rustup/toolchains/1.68.2-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/result.rs:507:5 + | +507 | Ok(#[stable(feature = "rust1", since = "1.0.0")] T), + | ^^ + +Some errors have detailed explanations: E0308, E0425, E0599. +For more information about an error, try `rustc --explain E0308`. +error: could not compile `compute_tools` due to 4 previous errors diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 21226249cf..fcef3e8723 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -30,3 +30,4 @@ url.workspace = true compute_api.workspace = true utils.workspace = true workspace_hack.workspace = true +remote_storage = { version = "0.1", path = "../libs/remote_storage/" } diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index cd5afa89f4..902a40e248 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -60,7 +60,7 @@ fn main() -> Result<()> { let matches = cli().get_matches(); let config = get_S3_config(matches); - download_extension(matches, ExtensionType::Shared); + download_extension(&matches, ExtensionType::Shared); let http_port = *matches .get_one::("http-port") @@ -206,7 +206,7 @@ fn main() -> Result<()> { let mut state = compute.state.lock().unwrap(); // Now we have the spec, and also the tenant id, so we can download the user's personal extensions - // download_extension(matches, ExtensionType::Tenant(FIXME tenant_id.into())); + // download_extension(&matches, ExtensionType::Tenant(FIXME tenant_id.into())); // Record for how long we slept waiting for the spec. state.metrics.wait_for_spec_ms = Utc::now() @@ -228,7 +228,7 @@ fn main() -> Result<()> { launch_configurator(&compute).expect("cannot launch configurator thread"); // Now we are ready to download library extensions - // download_extension(matches, ExtensionType::Library(FIXME library_name.into())); + // download_extension(&matches, ExtensionType::Library(FIXME library_name.into())); // Start Postgres let mut delay_exit = false; diff --git a/compute_tools/src/lib.rs b/compute_tools/src/lib.rs index 24811f75ee..9a3f4ebfb1 100644 --- a/compute_tools/src/lib.rs +++ b/compute_tools/src/lib.rs @@ -13,3 +13,4 @@ pub mod monitor; pub mod params; pub mod pg_helpers; pub mod spec; +pub mod pg_extensions; diff --git a/compute_tools/src/pg_extensions.rs b/compute_tools/src/pg_extensions.rs index d6a4cc2e3c..d034bba6cb 100644 --- a/compute_tools/src/pg_extensions.rs +++ b/compute_tools/src/pg_extensions.rs @@ -1,6 +1,8 @@ // This is some code for downloading postgres extensions from AWS S3 -use std::{env, ops::ControlFlow, path::Path, str::FromStr}; -use clap::{Arg, ArgAction, Command}; +use std::{ops::ControlFlow, path::Path}; +use clap::{ArgMatches}; +use tracing::*; +use remote_storage::*; fn get_pg_config(argument: &str) -> String { // NOTE: this function panics if it runs into any issues; @@ -15,13 +17,13 @@ fn get_pg_config(argument: &str) -> String { stdout.trim().to_string() } -fn download_helper(config: RemoteStorageConfig, from_path: &str, to_path: &str) -> anyhow::Result<()> { +fn download_helper(config: &RemoteStorageConfig, from_path: &str, to_path: &str) -> anyhow::Result<()> { let mut remote_storage = GenericRemoteStorage::from_config(config)?; - let data = remote_storage.download(remote_path); - // TODO: write "data" to "to_path" - println("received data"); - println!("{:?}", data); - Ok(()); + let remote_from_path = RemotePath::new(Path::new(from_path))?; + let data = remote_storage.download(&remote_from_path); + println!("received data, hopefully"); + Ok(()) + // TODO: somehow write "data" to "to_path" } pub enum ExtensionType { @@ -31,40 +33,41 @@ pub enum ExtensionType { } // TODO: should I make this async? -pub fn download_extension(config: RemoteStorageConfig, ext_type: ExtensionType) -> anyhow::Result<()>{ +pub fn download_extension(config: &RemoteStorageConfig, ext_type: ExtensionType) -> anyhow::Result<()>{ let sharedir = get_pg_config("--sharedir"); let sharedir = format!("{}/extension", sharedir); let libdir = get_pg_config("--libdir"); match ext_type { - Shared => { + ExtensionType::Shared => { // 1. Download control files from s3-bucket/public/*.control to SHAREDIR/extension // We can do this step even before we have spec, // because public extensions are common for all projects. - let from_path = "s3-bucket/public/*.control" - download_helper(config, from_path, sharedir); + let from_path = "s3-bucket/public/*.control"; + download_helper(config, from_path, &sharedir); } - Tenant(tenant_id) => { + ExtensionType::Tenant(tenant_id) => { // 2. After we have spec, before project start // Download control files from s3-bucket/[tenant-id]/*.control to SHAREDIR/extension let from_path = format!("s3-bucket/{tenant_id}/*.control"); - download_helper(config, from_path, sharedir); + download_helper(config, &from_path, &sharedir); } - Library(library_name) => { + ExtensionType::Library(library_name) => { // 3. After we have spec, before postgres start // Download preload_shared_libraries from s3-bucket/public/[library-name].control into LIBDIR/ let from_path = format!("s3-bucket/public/{library_name}.control"); - download_helper(config, from_path, libdir); + download_helper(config, &from_path, &libdir); } } + Ok(()) } -pub get_S3_config(arg_matches: ArgMatches) -> anyhow::Result { +pub fn get_S3_config(arg_matches: ArgMatches) -> anyhow::Result { let workdir = arg_matches .get_one::("workdir") .map(Path::new) - .unwrap_or_else(|| Path::new(".neon")) - .canonicalize() + .unwrap_or_else(|| Path::new(".neon")); + workdir = workdir.canonicalize() .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?; // TODO: is this the correct file location?