diff --git a/.gitignore b/.gitignore index e8e34b1c0c..d032a82441 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,5 @@ test_output/ *.o *.so *.Po + alek/ diff --git a/alek/env.txt b/alek/env.txt index 1d2bd28c2e..adaea2e185 100644 --- a/alek/env.txt +++ b/alek/env.txt @@ -1 +1 @@ -{'repo_dir': PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo'), 'rust_log_override': None, 'port_distributor': , 's3_mock_server': , 'neon_cli': , 'endpoints': , 'safekeepers': [Safekeeper(env=, port=SafekeeperPort(pg=15005, http=15006), id=1, running=True), Safekeeper(env=, port=SafekeeperPort(pg=15007, http=15008), id=2, running=True), Safekeeper(env=, port=SafekeeperPort(pg=15009, http=15010), id=3, running=True)], 'broker': NeonBroker(logfile=PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo/storage_broker.log'), port=15002, neon_binpath=PosixPath('/home/alek/Desktop/neonX/target/debug'), handle=), 'remote_storage': S3Storage(bucket_name='test_file_download', bucket_region='us-east-1', access_key='test', secret_key='test', endpoint='http://127.0.0.1:15001', prefix_in_bucket=None), 'remote_storage_users': , 'pg_version': '14', 'neon_binpath': PosixPath('/home/alek/Desktop/neonX/target/debug'), 'pg_distrib_dir': PosixPath('/home/alek/Desktop/neonX/pg_install'), 'endpoint_counter': 0, 'initial_tenant': `TenantId("258f07c42a24166e3c8ee13a6b6a665c"), 'initial_timeline': TimelineId("3dc8832f5f3bdca85aad1e190fd08582"), 'pageserver': } \ No newline at end of file +{'repo_dir': PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo'), 'rust_log_override': None, 'port_distributor': , 's3_mock_server': , 'neon_cli': , 'endpoints': , 'safekeepers': [Safekeeper(env=, port=SafekeeperPort(pg=15004, http=15005), id=1, running=True), Safekeeper(env=, port=SafekeeperPort(pg=15006, http=15007), id=2, running=True), Safekeeper(env=, port=SafekeeperPort(pg=15008, http=15009), id=3, running=True)], 'broker': NeonBroker(logfile=PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo/storage_broker.log'), port=15001, neon_binpath=PosixPath('/home/alek/Desktop/neonX/target/debug'), handle=), 'remote_storage': S3Storage(bucket_name='test_file_download', bucket_region='us-east-1', access_key='test', secret_key='test', endpoint='http://127.0.0.1:15000', prefix_in_bucket=None), 'remote_storage_users': , 'pg_version': '14', 'neon_binpath': PosixPath('/home/alek/Desktop/neonX/target/debug'), 'pg_distrib_dir': PosixPath('/home/alek/Desktop/neonX/pg_install'), 'endpoint_counter': 0, 'initial_tenant': `TenantId("15347163e7f9bb87079f0afbe58b2c5a"), 'initial_timeline': TimelineId("ac505335c89aa4e6c2e5b5660be3edd4"), 'pageserver': } \ No newline at end of file diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 63c6fc6d5d..ab012917b4 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -11,18 +11,18 @@ use tracing::info; pub async fn download_file( filename: &str, - // remote_ext_bucket: String, - // remote_ext_region: String, + remote_ext_bucket: String, + remote_ext_region: String, ) -> anyhow::Result<()> { // probably should be using the pgbin argv somehow to compute sharedir... - // let sharedir = get_pg_config("--sharedir"); - // fs::write("alek/sharedir.txt", sharedir)?; + let sharedir = get_pg_config("--sharedir"); + fs::write("alek/sharedir.txt", sharedir)?; println!("requested file {}", filename); // TODO: download the extensions! - // let s3_config = create_s3_config(remote_ext_bucket, remote_ext_region); - // download_extension(&s3_config, ExtensionType::Shared).await?; + let s3_config = create_s3_config(remote_ext_bucket, remote_ext_region); + download_extension(&s3_config, ExtensionType::Shared).await?; // This is filler code // let from_prefix = "/tmp/from_prefix"; diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index a8e39929f3..0b6e6f7bc0 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -38,6 +38,10 @@ fn status_response_from_state(state: &ComputeState) -> ComputeStatusResponse { // Service function to handle all available routes. async fn routes(req: Request, compute: &Arc) -> Response { + let xxx = format!("ROUTING{:?}", req.uri().path().clone()); + std::fs::write("alek/ROUTES", xxx).expect("routing write file"); + // TODO: NOTE TO SELF: these things are already being logged to compute.log + // // NOTE: The URI path is currently included in traces. That's OK because // it doesn't contain any variable parts or sensitive information. But @@ -138,8 +142,8 @@ async fn routes(req: Request, compute: &Arc) -> Response - static int extension_server_port = 0; static download_extension_file_hook_type prev_download_extension_file_hook = NULL; @@ -36,29 +35,29 @@ neon_download_extension_file_http(const char *filename) { CURL *curl; CURLcode res; - char * compute_ctl_url; + char *compute_ctl_url; char *postdata; bool ret = false; if ((curl = curl_easy_init()) == NULL) - { - elog(ERROR, "Failed to initialize curl handle"); - } + { + elog(ERROR, "Failed to initialize curl handle"); + } compute_ctl_url = psprintf("http://localhost:%d/extension_server/%s", extension_server_port, filename); elog(LOG, "curl_easy_perform() url: %s", compute_ctl_url); - curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); curl_easy_setopt(curl, CURLOPT_URL, compute_ctl_url); - curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L /* seconds */ ); + curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L /* seconds */); - if(curl) + if (curl) { /* Perform the request, res will get the return code */ res = curl_easy_perform(curl); /* Check for errors */ - if(res == CURLE_OK) + if (res == CURLE_OK) { elog(LOG, "curl_easy_perform() succeeded"); ret = true; @@ -75,19 +74,18 @@ neon_download_extension_file_http(const char *filename) return ret; } -void -pg_init_extension_server() +void pg_init_extension_server() { - DefineCustomIntVariable("neon.extension_server_port", - "connection string to the compute_ctl", - NULL, - &extension_server_port, - 0,0,INT_MAX, - PGC_POSTMASTER, - 0, /* no flags required */ - NULL, NULL, NULL); + DefineCustomIntVariable("neon.extension_server_port", + "connection string to the compute_ctl", + NULL, + &extension_server_port, + 0, 0, INT_MAX, + PGC_POSTMASTER, + 0, /* no flags required */ + NULL, NULL, NULL); - //set download_extension_file_hook + // set download_extension_file_hook prev_download_extension_file_hook = download_extension_file_hook; download_extension_file_hook = neon_download_extension_file_http; }