rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-16 01:42:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

more debugginh

Browse Source
This commit is contained in:
Alek Westover
2023-06-20 20:02:24 -04:00
parent 890061d371
commit 5e96ab43ea
5 changed files with 32 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -18,4 +18,5 @@ test_output/
*.o
*.so
*.Po
alek/

2
alek/env.txt
View File

@@ -1 +1 @@
{'repo_dir': PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo'), 'rust_log_override': None, 'port_distributor': <fixtures.neon_fixtures.PortDistributor object at 0x7efeeb54e890>, 's3_mock_server': <fixtures.neon_fixtures.MockS3Server object at 0x7efeeb54e950>, 'neon_cli': <fixtures.neon_fixtures.NeonCli object at 0x7efee9ac7a90>, 'endpoints': <fixtures.neon_fixtures.EndpointFactory object at 0x7efee9ac7a00>, 'safekeepers': [Safekeeper(env=<fixtures.neon_fixtures.NeonEnv object at 0x7efee9ac7a60>, port=SafekeeperPort(pg=15005, http=15006), id=1, running=True), Safekeeper(env=<fixtures.neon_fixtures.NeonEnv object at 0x7efee9ac7a60>, port=SafekeeperPort(pg=15007, http=15008), id=2, running=True), Safekeeper(env=<fixtures.neon_fixtures.NeonEnv object at 0x7efee9ac7a60>, port=SafekeeperPort(pg=15009, http=15010), id=3, running=True)], 'broker': NeonBroker(logfile=PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo/storage_broker.log'), port=15002, neon_binpath=PosixPath('/home/alek/Desktop/neonX/target/debug'), handle=<Popen: returncode: None args: ['/home/alek/Desktop/neonX/target/debug/stora...>), 'remote_storage': S3Storage(bucket_name='test_file_download', bucket_region='us-east-1', access_key='test', secret_key='test', endpoint='http://127.0.0.1:15001', prefix_in_bucket=None), 'remote_storage_users': <RemoteStorageUsers.PAGESERVER: 1>, 'pg_version': '14', 'neon_binpath': PosixPath('/home/alek/Desktop/neonX/target/debug'), 'pg_distrib_dir': PosixPath('/home/alek/Desktop/neonX/pg_install'), 'endpoint_counter': 0, 'initial_tenant': `TenantId("258f07c42a24166e3c8ee13a6b6a665c"), 'initial_timeline': TimelineId("3dc8832f5f3bdca85aad1e190fd08582"), 'pageserver': <fixtures.neon_fixtures.NeonPageserver object at 0x7efee9ac7910>}
{'repo_dir': PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo'), 'rust_log_override': None, 'port_distributor': <fixtures.neon_fixtures.PortDistributor object at 0x7faa9ed4a860>, 's3_mock_server': <fixtures.neon_fixtures.MockS3Server object at 0x7faa9ed4a920>, 'neon_cli': <fixtures.neon_fixtures.NeonCli object at 0x7faa9d2c3a30>, 'endpoints': <fixtures.neon_fixtures.EndpointFactory object at 0x7faa9d2c39a0>, 'safekeepers': [Safekeeper(env=<fixtures.neon_fixtures.NeonEnv object at 0x7faa9d2c3a00>, port=SafekeeperPort(pg=15004, http=15005), id=1, running=True), Safekeeper(env=<fixtures.neon_fixtures.NeonEnv object at 0x7faa9d2c3a00>, port=SafekeeperPort(pg=15006, http=15007), id=2, running=True), Safekeeper(env=<fixtures.neon_fixtures.NeonEnv object at 0x7faa9d2c3a00>, port=SafekeeperPort(pg=15008, http=15009), id=3, running=True)], 'broker': NeonBroker(logfile=PosixPath('/home/alek/Desktop/neonX/test_output/test_file_download[debug-pg14]/repo/storage_broker.log'), port=15001, neon_binpath=PosixPath('/home/alek/Desktop/neonX/target/debug'), handle=<Popen: returncode: None args: ['/home/alek/Desktop/neonX/target/debug/stora...>), 'remote_storage': S3Storage(bucket_name='test_file_download', bucket_region='us-east-1', access_key='test', secret_key='test', endpoint='http://127.0.0.1:15000', prefix_in_bucket=None), 'remote_storage_users': <RemoteStorageUsers.PAGESERVER: 1>, 'pg_version': '14', 'neon_binpath': PosixPath('/home/alek/Desktop/neonX/target/debug'), 'pg_distrib_dir': PosixPath('/home/alek/Desktop/neonX/pg_install'), 'endpoint_counter': 0, 'initial_tenant': `TenantId("15347163e7f9bb87079f0afbe58b2c5a"), 'initial_timeline': TimelineId("ac505335c89aa4e6c2e5b5660be3edd4"), 'pageserver': <fixtures.neon_fixtures.NeonPageserver object at 0x7faa9d2c38b0>}

12
compute_tools/src/extension_server.rs
View File

@@ -11,18 +11,18 @@ use tracing::info;
pub async fn download_file(
filename: &str,
// remote_ext_bucket: String,
// remote_ext_region: String,
remote_ext_bucket: String,
remote_ext_region: String,
) -> anyhow::Result<()> {
// probably should be using the pgbin argv somehow to compute sharedir...
// let sharedir = get_pg_config("--sharedir");
// fs::write("alek/sharedir.txt", sharedir)?;
let sharedir = get_pg_config("--sharedir");
fs::write("alek/sharedir.txt", sharedir)?;
println!("requested file {}", filename);
// TODO: download the extensions!
// let s3_config = create_s3_config(remote_ext_bucket, remote_ext_region);
// download_extension(&s3_config, ExtensionType::Shared).await?;
let s3_config = create_s3_config(remote_ext_bucket, remote_ext_region);
download_extension(&s3_config, ExtensionType::Shared).await?;
// This is filler code
// let from_prefix = "/tmp/from_prefix";

8
compute_tools/src/http/api.rs
View File

@@ -38,6 +38,10 @@ fn status_response_from_state(state: &ComputeState) -> ComputeStatusResponse {
// Service function to handle all available routes.
async fn routes(req: Request<Body>, compute: &Arc<ComputeNode>) -> Response<Body> {
let xxx = format!("ROUTING{:?}", req.uri().path().clone());
std::fs::write("alek/ROUTES", xxx).expect("routing write file");
// TODO: NOTE TO SELF: these things are already being logged to compute.log
//
// NOTE: The URI path is currently included in traces. That's OK because
// it doesn't contain any variable parts or sensitive information. But
@@ -138,8 +142,8 @@ async fn routes(req: Request<Body>, compute: &Arc<ComputeNode>) -> Response<Body
match extension_server::download_file(
filename,
// TODO alek: pass more remote_ext arguments
// compute.remote_ext_bucket.clone(),
// compute.remote_ext_region.clone(),
compute.remote_ext_bucket.clone(),
compute.remote_ext_region.clone(),
)
.await
{

38
pgxn/neon/extension_server.c
View File

@@ -25,7 +25,6 @@
#include <curl/curl.h>
static int extension_server_port = 0;
static download_extension_file_hook_type prev_download_extension_file_hook = NULL;
@@ -36,29 +35,29 @@ neon_download_extension_file_http(const char *filename)
{
CURL *curl;
CURLcode res;
char * compute_ctl_url;
char *compute_ctl_url;
char *postdata;
bool ret = false;
if ((curl = curl_easy_init()) == NULL)
{
elog(ERROR, "Failed to initialize curl handle");
}
{
elog(ERROR, "Failed to initialize curl handle");
}
compute_ctl_url = psprintf("http://localhost:%d/extension_server/%s", extension_server_port, filename);
elog(LOG, "curl_easy_perform() url: %s", compute_ctl_url);
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST");
curl_easy_setopt(curl, CURLOPT_URL, compute_ctl_url);
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L /* seconds */ );
curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3L /* seconds */);
if(curl)
if (curl)
{
/* Perform the request, res will get the return code */
res = curl_easy_perform(curl);
/* Check for errors */
if(res == CURLE_OK)
if (res == CURLE_OK)
{
elog(LOG, "curl_easy_perform() succeeded");
ret = true;
@@ -75,19 +74,18 @@ neon_download_extension_file_http(const char *filename)
return ret;
}
void
pg_init_extension_server()
void pg_init_extension_server()
{
DefineCustomIntVariable("neon.extension_server_port",
"connection string to the compute_ctl",
NULL,
&extension_server_port,
0,0,INT_MAX,
PGC_POSTMASTER,
0, /* no flags required */
NULL, NULL, NULL);
DefineCustomIntVariable("neon.extension_server_port",
"connection string to the compute_ctl",
NULL,
&extension_server_port,
0, 0, INT_MAX,
PGC_POSTMASTER,
0, /* no flags required */
NULL, NULL, NULL);
//set download_extension_file_hook
// set download_extension_file_hook
prev_download_extension_file_hook = download_extension_file_hook;
download_extension_file_hook = neon_download_extension_file_http;
}