mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-06 04:52:55 +00:00
code cleanup for compute_node_rebase branch
This commit is contained in:
anastasia
@@ -70,12 +70,9 @@ impl StorageControlPlane {
|
||||
data_dir: TEST_WORKDIR.join("pageserver"),
|
||||
};
|
||||
pserver.init();
|
||||
if froms3
|
||||
{
|
||||
if froms3 {
|
||||
pserver.start_froms3();
|
||||
}
|
||||
else
|
||||
{
|
||||
} else {
|
||||
pserver.start();
|
||||
}
|
||||
|
||||
@@ -379,10 +376,10 @@ impl ComputeControlPlane<'_> {
|
||||
node
|
||||
}
|
||||
|
||||
// Init compute node without files, only datadir structure
|
||||
// use initdb --compute-node flag and GUC 'computenode_mode'
|
||||
// to distinguish the node
|
||||
pub fn new_minimal_node<'a>(&mut self) -> &Arc<PostgresNode> {
|
||||
// Init compute node without files, only datadir structure
|
||||
// use initdb --compute-node flag and GUC 'computenode_mode'
|
||||
// to distinguish the node
|
||||
pub fn new_minimal_node<'a>(&mut self) -> &Arc<PostgresNode> {
|
||||
// allocate new node entry with generated port
|
||||
let node_id = self.nodes.len() + 1;
|
||||
let node = PostgresNode {
|
||||
@@ -448,9 +445,17 @@ impl ComputeControlPlane<'_> {
|
||||
let pserver = storage_cplane.page_server_addr();
|
||||
|
||||
// Configure that node to take pages from pageserver
|
||||
node.append_conf("postgresql.conf", format!("\
|
||||
node.append_conf(
|
||||
"postgresql.conf",
|
||||
format!(
|
||||
"\
|
||||
page_server_connstring = 'host={} port={}'\n\
|
||||
", pserver.ip(), pserver.port()).as_str());
|
||||
",
|
||||
pserver.ip(),
|
||||
pserver.port()
|
||||
)
|
||||
.as_str(),
|
||||
);
|
||||
|
||||
node.clone()
|
||||
}
|
||||
@@ -525,7 +530,6 @@ pub struct PostgresNode {
|
||||
pg_bin_dir: PathBuf,
|
||||
}
|
||||
|
||||
|
||||
impl PostgresNode {
|
||||
pub fn append_conf(&self, config: &str, opts: &str) {
|
||||
OpenOptions::new()
|
||||
@@ -636,80 +640,81 @@ impl PostgresNode {
|
||||
// And reqular query() uses prepared queries.
|
||||
|
||||
// TODO pass sysid as parameter
|
||||
pub fn setup_compute_node(&self, sysid: u64, storage_cplane: &StorageControlPlane)
|
||||
{
|
||||
pub fn setup_compute_node(&self, sysid: u64, storage_cplane: &StorageControlPlane) {
|
||||
let mut query;
|
||||
//Request pg_control from pageserver
|
||||
query = format!("file {}/global/pg_control,{},{},{},{},{},{},{}",
|
||||
query = format!(
|
||||
"file {}/global/pg_control,{},{},{},{},{},{},{}",
|
||||
self.pgdata.to_str().unwrap(),
|
||||
sysid as u64, //sysid
|
||||
1664, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
42, //forknum pg_control
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
1664, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
42, //forknum pg_control
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
);
|
||||
storage_cplane.page_server_psql(query.as_str());
|
||||
|
||||
//Request pg_xact and pg_multixact from pageserver
|
||||
//We need them for initial pageserver startup and authentication
|
||||
//TODO figure out which block number we really need
|
||||
query = format!("file {}/pg_xact/0000,{},{},{},{},{},{},{}",
|
||||
query = format!(
|
||||
"file {}/pg_xact/0000,{},{},{},{},{},{},{}",
|
||||
self.pgdata.to_str().unwrap(),
|
||||
sysid as u64, //sysid
|
||||
0, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
44, //forknum
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
0, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
44, //forknum
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
);
|
||||
storage_cplane.page_server_psql(query.as_str());
|
||||
|
||||
query = format!("file {}/pg_multixact/offsets/0000,{},{},{},{},{},{},{}",
|
||||
query = format!(
|
||||
"file {}/pg_multixact/offsets/0000,{},{},{},{},{},{},{}",
|
||||
self.pgdata.to_str().unwrap(),
|
||||
sysid as u64, //sysid
|
||||
0, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
45, //forknum
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
0, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
45, //forknum
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
);
|
||||
storage_cplane.page_server_psql(query.as_str());
|
||||
|
||||
query = format!("file {}/pg_multixact/members/0000,{},{},{},{},{},{},{}",
|
||||
query = format!(
|
||||
"file {}/pg_multixact/members/0000,{},{},{},{},{},{},{}",
|
||||
self.pgdata.to_str().unwrap(),
|
||||
sysid as u64, //sysid
|
||||
0, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
46, //forknum
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
0, //tablespace
|
||||
0, //dboid
|
||||
0, //reloid
|
||||
46, //forknum
|
||||
0, //blkno
|
||||
0 //lsn
|
||||
);
|
||||
storage_cplane.page_server_psql(query.as_str());
|
||||
|
||||
|
||||
//Request a few shared catalogs needed for authentication
|
||||
//Without them we cannot setup connection with pageserver to request further pages
|
||||
let reloids = [1260, 1261, 1262, 2396];
|
||||
for reloid in reloids.iter()
|
||||
{
|
||||
for reloid in reloids.iter() {
|
||||
//FIXME request all blocks from file, not just 10
|
||||
for blkno in 0..10
|
||||
{
|
||||
query = format!("file {}/global/{},{},{},{},{},{},{},{}",
|
||||
for blkno in 0..10 {
|
||||
query = format!(
|
||||
"file {}/global/{},{},{},{},{},{},{},{}",
|
||||
self.pgdata.to_str().unwrap(),
|
||||
reloid, //suse it as filename
|
||||
reloid, //suse it as filename
|
||||
sysid as u64, //sysid
|
||||
1664, //tablespace
|
||||
0, //dboid
|
||||
reloid, //reloid
|
||||
0, //forknum
|
||||
blkno, //blkno
|
||||
0 //lsn
|
||||
1664, //tablespace
|
||||
0, //dboid
|
||||
reloid, //reloid
|
||||
0, //forknum
|
||||
blkno, //blkno
|
||||
0 //lsn
|
||||
);
|
||||
storage_cplane.page_server_psql(query.as_str());
|
||||
}
|
||||
@@ -719,8 +724,15 @@ impl PostgresNode {
|
||||
fs::create_dir(format!("{}/base/13007", self.pgdata.to_str().unwrap())).unwrap();
|
||||
|
||||
//FIXME figure out what wal file we need to successfully start
|
||||
let walfilepath = format!("{}/pg_wal/000000010000000000000001", self.pgdata.to_str().unwrap());
|
||||
fs::copy("/home/anastasia/zenith/zenith/tmp_check/pgdata/pg_wal/000000010000000000000001", walfilepath).unwrap();
|
||||
let walfilepath = format!(
|
||||
"{}/pg_wal/000000010000000000000001",
|
||||
self.pgdata.to_str().unwrap()
|
||||
);
|
||||
fs::copy(
|
||||
"/home/anastasia/zenith/zenith/tmp_check/pgdata/pg_wal/000000010000000000000001",
|
||||
walfilepath,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
println!("before resetwal ");
|
||||
|
||||
@@ -743,7 +755,6 @@ impl PostgresNode {
|
||||
}
|
||||
|
||||
println!("setup done");
|
||||
|
||||
}
|
||||
|
||||
pub fn start_proxy(&self, wal_acceptors: String) -> WalProposerNode {
|
||||
@@ -761,8 +772,7 @@ impl PostgresNode {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_to_s3(&self)
|
||||
{
|
||||
pub fn push_to_s3(&self) {
|
||||
println!("Push to s3 node at '{}'", self.pgdata.to_str().unwrap());
|
||||
|
||||
let zenith_push_path = self.pg_bin_dir.join("zenith_push");
|
||||
|
||||
@@ -6,7 +6,6 @@ use std::time::Duration;
|
||||
use control_plane::ComputeControlPlane;
|
||||
use control_plane::StorageControlPlane;
|
||||
|
||||
|
||||
// XXX: force all redo at the end
|
||||
// -- restart + seqscan won't read deleted stuff
|
||||
// -- pageserver api endpoint to check all rels
|
||||
@@ -27,8 +26,14 @@ fn test_redo_cases() {
|
||||
sleep(Duration::from_secs(3));
|
||||
|
||||
// check basic work with table
|
||||
node.safe_psql("postgres", "CREATE TABLE t(key int primary key, value text)");
|
||||
node.safe_psql("postgres", "INSERT INTO t SELECT generate_series(1,100), 'payload'");
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"CREATE TABLE t(key int primary key, value text)",
|
||||
);
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"INSERT INTO t SELECT generate_series(1,100), 'payload'",
|
||||
);
|
||||
let count: i64 = node
|
||||
.safe_psql("postgres", "SELECT sum(key) FROM t")
|
||||
.first()
|
||||
@@ -86,8 +91,14 @@ fn test_pageserver_multitenancy() {
|
||||
sleep(Duration::from_secs(3));
|
||||
|
||||
// check node1
|
||||
node1.safe_psql("postgres", "CREATE TABLE t(key int primary key, value text)");
|
||||
node1.safe_psql("postgres", "INSERT INTO t SELECT generate_series(1,100), 'payload'");
|
||||
node1.safe_psql(
|
||||
"postgres",
|
||||
"CREATE TABLE t(key int primary key, value text)",
|
||||
);
|
||||
node1.safe_psql(
|
||||
"postgres",
|
||||
"INSERT INTO t SELECT generate_series(1,100), 'payload'",
|
||||
);
|
||||
let count: i64 = node1
|
||||
.safe_psql("postgres", "SELECT sum(key) FROM t")
|
||||
.first()
|
||||
@@ -97,8 +108,14 @@ fn test_pageserver_multitenancy() {
|
||||
assert_eq!(count, 5050);
|
||||
|
||||
// check node2
|
||||
node2.safe_psql("postgres", "CREATE TABLE t(key int primary key, value text)");
|
||||
node2.safe_psql("postgres", "INSERT INTO t SELECT generate_series(100,200), 'payload'");
|
||||
node2.safe_psql(
|
||||
"postgres",
|
||||
"CREATE TABLE t(key int primary key, value text)",
|
||||
);
|
||||
node2.safe_psql(
|
||||
"postgres",
|
||||
"INSERT INTO t SELECT generate_series(100,200), 'payload'",
|
||||
);
|
||||
let count: i64 = node2
|
||||
.safe_psql("postgres", "SELECT sum(key) FROM t")
|
||||
.first()
|
||||
@@ -120,7 +137,8 @@ fn test_pageserver_multitenancy() {
|
||||
// .env("S3_BUCKET", "zenith-testbucket")
|
||||
// TODO use env variables in test
|
||||
fn test_pageserver_recovery() {
|
||||
|
||||
//This test expects that image is already uploaded to s3
|
||||
//To upload it use zenith_push before test (see node.push_to_s3() for details)
|
||||
let storage_cplane = StorageControlPlane::one_page_server(true);
|
||||
let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
|
||||
|
||||
@@ -143,7 +161,6 @@ fn test_pageserver_recovery() {
|
||||
#[ignore]
|
||||
//Scenario for future test. Not implemented yet
|
||||
fn test_pageserver_node_switch() {
|
||||
|
||||
//Create pageserver
|
||||
let storage_cplane = StorageControlPlane::one_page_server(false);
|
||||
let mut compute_cplane = ComputeControlPlane::local(&storage_cplane);
|
||||
@@ -152,8 +169,14 @@ fn test_pageserver_node_switch() {
|
||||
let node = compute_cplane.new_node();
|
||||
node.start(&storage_cplane);
|
||||
|
||||
node.safe_psql("postgres", "CREATE TABLE t(key int primary key, value text)");
|
||||
node.safe_psql("postgres", "INSERT INTO t SELECT generate_series(1,100), 'payload'");
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"CREATE TABLE t(key int primary key, value text)",
|
||||
);
|
||||
node.safe_psql(
|
||||
"postgres",
|
||||
"INSERT INTO t SELECT generate_series(1,100), 'payload'",
|
||||
);
|
||||
let count: i64 = node
|
||||
.safe_psql("postgres", "SELECT sum(key) FROM t")
|
||||
.first()
|
||||
|
||||
Reference in New Issue
Block a user