rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-08-18 03:58:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Bojan Serafimov
2023-06-08 10:21:07 -04:00
parent eda4f86588
commit 881bfc4da8
4 changed files with 145 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+1
View File
@@ -108,6 +108,7 @@ postgres-%: postgres-configure-% \
$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pg_buffercache install
+@echo "Compiling pageinspect $*"
$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pageinspect install
$(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/$*/contrib/pg_stat_statements install
.PHONY: postgres-clean-%
postgres-clean-%:
compute_tools/src/compute.rs
+36 -1
View File
@@ -442,8 +442,43 @@ impl ComputeNode {
let pg = self.start_postgres(spec.storage_auth_token.clone())?;
// Maybe apply the spec
if spec.spec.mode == ComputeMode::Primary {
self.apply_config(&compute_state)?;
let spec = &compute_state.pspec.as_ref().expect("spec must be set").spec;
// Get spec_id or make it up by hashing
//
// TODO Make spec_id required so there would be no need to hash.
let spec_id = spec.operation_uuid.clone().unwrap_or_else(|| {
use std::collections::hash_map::DefaultHasher;
use std::hash::{Hash, Hasher};
// HACK Exclude postgresql.conf because it doesn't need
// to be applied like the other fields in the spec
let mut spec_no_conf = spec.clone();
spec_no_conf.cluster.postgresql_conf = None;
let json = serde_json::to_vec(&spec_no_conf).unwrap();
let mut hasher = DefaultHasher::new();
json.hash(&mut hasher);
let hash = hasher.finish();
format!("{:x}", hash)
});
// Get current spec_id
// TODO use pageserver instead of local storage
let path = Path::new("/home/bojan/tmp/spec_id.txt");
let current_spec_id = std::fs::read_to_string(path).ok();
// Respec if needed
if current_spec_id == Some(spec_id.clone()) {
info!("no need to respec");
} else {
info!("respeccing {:?} {:?}", current_spec_id, spec_id.clone());
self.apply_config(&compute_state)?;
std::fs::write(path, spec_id)?;
}
}
let startup_end_time = Utc::now();
control_plane/src/endpoint.rs
+84 -3
View File
@@ -43,6 +43,9 @@ use std::sync::Arc;
use std::time::Duration;
use anyhow::{anyhow, bail, Context, Result};
use compute_api::spec::Database;
use compute_api::spec::GenericOption;
use compute_api::spec::Role;
use serde::{Deserialize, Serialize};
use serde_with::{serde_as, DisplayFromStr};
use utils::id::{NodeId, TenantId, TimelineId};
@@ -456,9 +459,87 @@ impl Endpoint {
cluster_id: None, // project ID: not used
name: None, // project name: not used
state: None,
roles: vec![],
databases: vec![],
settings: None,
// TODO pass this info from the test
roles: vec![
Role {
name: "cloud_admin".into(),
encrypted_password: None,
options: None,
},
Role {
name: "foo".into(),
encrypted_password: Some("bar".into()),
options: None,
},
Role {
name: "foo2".into(),
encrypted_password: Some("bar2".into()),
options: None,
},
Role {
name: "foo3".into(),
encrypted_password: Some("bar3".into()),
options: None,
},
Role {
name: "foo4".into(),
encrypted_password: Some("bar4".into()),
options: None,
},
],
databases: vec![
Database {
name: "postgres".into(),
owner: "cloud_admin".into(),
options: None,
},
Database {
name: "postgres_2".into(),
owner: "cloud_admin".into(),
options: None,
},
Database {
name: "postgres_3".into(),
owner: "cloud_admin".into(),
options: None,
},
Database {
name: "postgres_4".into(),
owner: "cloud_admin".into(),
options: None,
},
Database {
name: "postgres_5".into(),
owner: "cloud_admin".into(),
options: None,
},
Database {
name: "postgres_6".into(),
owner: "cloud_admin".into(),
options: None,
},
Database {
name: "postgres_7".into(),
owner: "cloud_admin".into(),
options: None,
},
Database {
name: "postgres_8".into(),
owner: "cloud_admin".into(),
options: None,
},
],
settings: Some(vec![
GenericOption {
name: "shared_preload_libraries".into(),
value: Some("neon,pg_stat_statements".into()),
// TODO test with this larger list of extensions. But first they
// need to be built (see compute dockerfile).
//
// value: Some("neon,pg_stat_statements,timescaledb,pg_cron".into()),
vartype: "string".into(),
},
]),
postgresql_conf: Some(postgresql_conf),
},
delta_operations: None,
test_runner/performance/test_startup.py
+24 -14
View File
@@ -28,21 +28,31 @@ def test_startup_simple(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenc
env = neon_env_builder.init_start()
env.neon_cli.create_branch("test_startup")
with zenbenchmark.record_duration("start_and_select"):
endpoint = env.endpoints.create_start("test_startup")
endpoint.safe_psql("select 1;")
metrics = requests.get(f"http://localhost:{endpoint.http_port}/metrics.json").json()
durations = {
"wait_for_spec_ms": "wait_for_spec",
"sync_safekeepers_ms": "sync_safekeepers",
"basebackup_ms": "basebackup",
"config_ms": "config",
"total_startup_ms": "total_startup",
}
for key, name in durations.items():
value = metrics[key]
zenbenchmark.record(name, value, "ms", report=MetricReport.LOWER_IS_BETTER)
for i in range(2):
# Start
with zenbenchmark.record_duration(f"{i}_start_and_select"):
endpoint = env.endpoints.create_start("test_startup", config_lines=[
# "shared_preload_libraries='neon,pg_stat_statements,timescaledb,pg_cron'"])
"shared_preload_libraries='neon,pg_stat_statements'"])
endpoint.safe_psql("select 1;")
# Get metrics
metrics = requests.get(f"http://localhost:{endpoint.http_port}/metrics.json").json()
durations = {
"wait_for_spec_ms": f"{i}_wait_for_spec",
"sync_safekeepers_ms": f"{i}_sync_safekeepers",
"basebackup_ms": f"{i}_basebackup",
"config_ms": f"{i}_config",
"total_startup_ms": f"{i}_total_startup",
}
for key, name in durations.items():
value = metrics[key]
zenbenchmark.record(name, value, "ms", report=MetricReport.LOWER_IS_BETTER)
# Stop so we can restart
endpoint.stop()
# This test sometimes runs for longer than the global 5 minute timeout.