From 59163cf3b3a63d938e0bbea2e171ce3aa3a3f484 Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Sat, 10 Apr 2021 11:55:50 +0300 Subject: [PATCH 01/33] Rework controle_plane code to reuse it in CLI. Move all paths from control_plane to local_env which can set them for testing environment or for local installation. --- .gitignore | 2 +- Cargo.lock | 42 ++ Cargo.toml | 2 + control_plane/.gitignore | 1 + control_plane/Cargo.toml | 20 + .../mod.rs => control_plane/src/lib.rs | 382 +++++++++--------- control_plane/src/local_env.rs | 210 ++++++++++ integration_tests/.gitignore | 1 + integration_tests/Cargo.toml | 1 + integration_tests/tests/test_pageserver.rs | 26 +- integration_tests/tests/test_wal_acceptor.rs | 54 +-- zenith/Cargo.toml | 11 + zenith/src/main.rs | 119 ++++++ 13 files changed, 648 insertions(+), 223 deletions(-) create mode 100644 control_plane/.gitignore create mode 100644 control_plane/Cargo.toml rename integration_tests/tests/control_plane/mod.rs => control_plane/src/lib.rs (69%) create mode 100644 control_plane/src/local_env.rs create mode 100644 integration_tests/.gitignore create mode 100644 zenith/Cargo.toml create mode 100644 zenith/src/main.rs diff --git a/.gitignore b/.gitignore index 2f22547efd..20348359a4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ /target -/tmp_check +/tmp_check/ /tmp_install diff --git a/Cargo.lock b/Cargo.lock index 0ac61eb60d..04d3842934 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -378,6 +378,21 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "control_plane" +version = "0.1.0" +dependencies = [ + "home", + "pageserver", + "postgres", + "rand 0.8.3", + "serde", + "serde_derive", + "tokio-postgres", + "toml", + "walkeeper", +] + [[package]] name = "core-foundation" version = "0.9.1" @@ -795,6 +810,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2456aef2e6b6a9784192ae780c0f15bc57df0e918585282325e8c8ac27737654" +dependencies = [ + "winapi", +] + [[package]] name = "http" version = "0.2.3" @@ -900,6 +924,7 @@ dependencies = [ name = "integration_tests" version = "0.1.0" dependencies = [ + "control_plane", "lazy_static", "pageserver", "postgres", @@ -2094,6 +2119,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +dependencies = [ + "serde", +] + [[package]] name = "tower-service" version = "0.3.1" @@ -2417,3 +2451,11 @@ name = "xml-rs" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" + +[[package]] +name = "zenith" +version = "0.1.0" +dependencies = [ + "clap", + "control_plane", +] diff --git a/Cargo.toml b/Cargo.toml index b6809e2ad7..f4d6314283 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,4 +3,6 @@ members = [ "integration_tests", "pageserver", "walkeeper", + "zenith", + "control_plane", ] diff --git a/control_plane/.gitignore b/control_plane/.gitignore new file mode 100644 index 0000000000..c1e54a6bcb --- /dev/null +++ b/control_plane/.gitignore @@ -0,0 +1 @@ +tmp_check/ diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml new file mode 100644 index 0000000000..dac78ea356 --- /dev/null +++ b/control_plane/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "control_plane" +version = "0.1.0" +authors = ["Stas Kelvich "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +rand = "0.8.3" +postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } +tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } + +serde = "" +serde_derive = "" +toml = "" +home = "0.5.3" + +pageserver = { path = "../pageserver" } +walkeeper = { path = "../walkeeper" } diff --git a/integration_tests/tests/control_plane/mod.rs b/control_plane/src/lib.rs similarity index 69% rename from integration_tests/tests/control_plane/mod.rs rename to control_plane/src/lib.rs index eab3f345af..09e172ec4a 100644 --- a/integration_tests/tests/control_plane/mod.rs +++ b/control_plane/src/lib.rs @@ -9,76 +9,68 @@ use std::fs::File; use std::fs::{self, OpenOptions}; +use std::net::TcpStream; use std::path::{Path, PathBuf}; use std::process::Command; use std::str; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::thread; +use std::time::Duration; use std::{ io::Write, net::{IpAddr, Ipv4Addr, SocketAddr}, }; -use lazy_static::lazy_static; +pub mod local_env; +use local_env::LocalEnv; use postgres::{Client, NoTls}; -lazy_static! { - // postgres would be there if it was build by 'make postgres' here in the repo - pub static ref PG_BIN_DIR : PathBuf = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../tmp_install/bin"); - pub static ref PG_LIB_DIR : PathBuf = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("../tmp_install/lib"); - - pub static ref BIN_DIR : PathBuf = cargo_bin_dir(); - - pub static ref TEST_WORKDIR : PathBuf = Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tmp_check"); -} - -// Find the directory where the binaries were put (i.e. target/debug/) -pub fn cargo_bin_dir() -> PathBuf { - let mut pathbuf = std::env::current_exe().ok().unwrap(); - - pathbuf.pop(); - if pathbuf.ends_with("deps") { - pathbuf.pop(); - } - - return pathbuf; -} - +// +// Collection of several example deployments useful for tests. // // I'm intendedly modelling storage and compute control planes as a separate entities // as it is closer to the actual setup. // -pub struct StorageControlPlane { +pub struct TestStorageControlPlane { pub wal_acceptors: Vec, - pub page_servers: Vec, + pub pageserver: Arc, + pub test_done: AtomicBool, } -impl StorageControlPlane { +impl TestStorageControlPlane { // postgres <-> page_server - pub fn one_page_server() -> StorageControlPlane { - let mut cplane = StorageControlPlane { - wal_acceptors: Vec::new(), - page_servers: Vec::new(), - }; + pub fn one_page_server() -> TestStorageControlPlane { + let env = local_env::test_env(); - let pserver = PageServerNode { - page_service_addr: "127.0.0.1:65200".parse().unwrap(), - data_dir: TEST_WORKDIR.join("pageserver"), - }; + let pserver = Arc::new(PageServerNode { + env: env.clone(), + kill_on_exit: true, + }); pserver.init(); pserver.start(); - cplane.page_servers.push(pserver); - cplane + TestStorageControlPlane { + wal_acceptors: Vec::new(), + pageserver: pserver, + test_done: AtomicBool::new(false), + } } - pub fn fault_tolerant(redundancy: usize) -> StorageControlPlane { - let mut cplane = StorageControlPlane { + // postgres <-> {wal_acceptor1, wal_acceptor2, ...} + pub fn fault_tolerant(redundancy: usize) -> TestStorageControlPlane { + let env = local_env::test_env(); + let mut cplane = TestStorageControlPlane { wal_acceptors: Vec::new(), - page_servers: Vec::new(), + pageserver: Arc::new(PageServerNode { + env: env.clone(), + kill_on_exit: true, + }), + test_done: AtomicBool::new(false), }; + cplane.pageserver.init(); + cplane.pageserver.start(); + const WAL_ACCEPTOR_PORT: usize = 54321; for i in 0..redundancy { @@ -86,7 +78,8 @@ impl StorageControlPlane { listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i) .parse() .unwrap(), - data_dir: TEST_WORKDIR.join(format!("wal_acceptor_{}", i)), + data_dir: env.data_dir.join(format!("wal_acceptor_{}", i)), + env: env.clone(), }; wal_acceptor.init(); wal_acceptor.start(); @@ -96,17 +89,7 @@ impl StorageControlPlane { } pub fn stop(&self) { - for wa in self.wal_acceptors.iter() { - wa.stop(); - } - } - - // // postgres <-> wal_acceptor x3 <-> page_server - // fn local(&mut self) -> StorageControlPlane { - // } - - pub fn page_server_addr(&self) -> &SocketAddr { - &self.page_servers[0].page_service_addr + self.test_done.store(true, Ordering::Relaxed); } pub fn get_wal_acceptor_conn_info(&self) -> String { @@ -117,13 +100,99 @@ impl StorageControlPlane { .join(",") } + pub fn is_running(&self) -> bool { + self.test_done.load(Ordering::Relaxed) + } +} + +impl Drop for TestStorageControlPlane { + fn drop(&mut self) { + self.stop(); + } +} + +// +// Control routines for pageserver. +// +// Used in CLI and tests. +// +pub struct PageServerNode { + kill_on_exit: bool, + env: LocalEnv, +} + +impl PageServerNode { + pub fn init(&self) { + fs::create_dir_all(self.env.pageserver_data_dir()).unwrap(); + } + + pub fn start(&self) { + println!( + "Starting pageserver at '{}'", + self.env.pageserver.listen_address + ); + + let status = Command::new(self.env.zenith_distrib_dir.join("pageserver")) // XXX -> method + .args(&["-D", self.env.pageserver_data_dir().to_str().unwrap()]) + .args(&[ + "-l", + self.env.pageserver.listen_address.to_string().as_str(), + ]) + .arg("-d") + .arg("--skip-recovery") + .env_clear() + .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) + .status() + .expect("failed to start pageserver"); + + if !status.success() { + panic!("pageserver start failed"); + } + } + + pub fn stop(&self) { + let pidfile = self.env.pageserver_pidfile(); + let pid = fs::read_to_string(pidfile).unwrap(); + + let status = Command::new("kill") + .arg(pid) + .env_clear() + .status() + .expect("failed to execute kill"); + + if !status.success() { + panic!("kill start failed"); + } + + // await for pageserver stop + for _ in 0..5 { + let stream = TcpStream::connect(self.env.pageserver.listen_address); + if let Err(_e) = stream { + return; + } + println!( + "Stopping pageserver on {}", + self.env.pageserver.listen_address + ); + thread::sleep(Duration::from_secs(1)); + } + + // ok, we failed to stop pageserver, let's panic + panic!("Failed to stop pageserver"); + } + + pub fn address(&self) -> &std::net::SocketAddr { + &self.env.pageserver.listen_address + } + pub fn page_server_psql(&self, sql: &str) -> Vec { - let addr = &self.page_servers[0].page_service_addr; + // let addr = &self.page_servers[0].env.pageserver.listen_address; let connstring = format!( "host={} port={} dbname={} user={}", - addr.ip(), - addr.port(), + self.address().ip(), + self.address().port(), "no_db", "no_user", ); @@ -134,83 +203,23 @@ impl StorageControlPlane { } } -impl Drop for StorageControlPlane { - fn drop(&mut self) { - self.stop(); - } -} - -pub struct PageServerNode { - page_service_addr: SocketAddr, - data_dir: PathBuf, -} - -impl PageServerNode { - // TODO: method to force redo on a specific relation - - // TODO: make wal-redo-postgres workable without data directory? - pub fn init(&self) { - fs::create_dir_all(self.data_dir.clone()).unwrap(); - - let datadir_path = self.data_dir.join("wal_redo_pgdata"); - fs::remove_dir_all(datadir_path.to_str().unwrap()).ok(); - - let initdb = Command::new(PG_BIN_DIR.join("initdb")) - .args(&["-D", datadir_path.to_str().unwrap()]) - .arg("-N") - .arg("--no-instructions") - .env_clear() - .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap()) - .status() - .expect("failed to execute initdb"); - if !initdb.success() { - panic!("initdb failed"); - } - } - - pub fn start(&self) { - println!("Starting pageserver at '{}'", self.page_service_addr); - - let status = Command::new(BIN_DIR.join("pageserver")) - .args(&["-D", self.data_dir.to_str().unwrap()]) - .args(&["-l", self.page_service_addr.to_string().as_str()]) - .arg("-d") - .arg("--skip-recovery") - .env_clear() - .env("PATH", PG_BIN_DIR.to_str().unwrap()) // path to postres-wal-redo binary - .status() - .expect("failed to start pageserver"); - - if !status.success() { - panic!("pageserver start failed"); - } - } - - pub fn stop(&self) { - let pidfile = self.data_dir.join("pageserver.pid"); - let pid = fs::read_to_string(pidfile).unwrap(); - let status = Command::new("kill") - .arg(pid) - .env_clear() - .status() - .expect("failed to execute kill"); - - if !status.success() { - panic!("kill start failed"); - } - } -} - impl Drop for PageServerNode { fn drop(&mut self) { - self.stop(); - // fs::remove_dir_all(self.data_dir.clone()).unwrap(); + if self.kill_on_exit { + self.stop(); + } } } +// +// Control routines for WalAcceptor. +// +// Now used only in test setups. +// pub struct WalAcceptorNode { listen: SocketAddr, data_dir: PathBuf, + env: LocalEnv, } impl WalAcceptorNode { @@ -228,7 +237,7 @@ impl WalAcceptorNode { self.listen ); - let status = Command::new(BIN_DIR.join("wal_acceptor")) + let status = Command::new(self.env.zenith_distrib_dir.join("wal_acceptor")) .args(&["-D", self.data_dir.to_str().unwrap()]) .args(&["-l", self.listen.to_string().as_str()]) .arg("-d") @@ -242,6 +251,7 @@ impl WalAcceptorNode { } pub fn stop(&self) { + println!("Stopping wal acceptor on {}", self.listen); let pidfile = self.data_dir.join("wal_acceptor.pid"); if let Ok(pid) = fs::read_to_string(pidfile) { let _status = Command::new("kill") @@ -256,7 +266,6 @@ impl WalAcceptorNode { impl Drop for WalAcceptorNode { fn drop(&mut self) { self.stop(); - // fs::remove_dir_all(self.data_dir.clone()).unwrap(); } } @@ -265,22 +274,25 @@ impl Drop for WalAcceptorNode { // // ComputeControlPlane // -pub struct ComputeControlPlane<'a> { +pub struct ComputeControlPlane { pg_bin_dir: PathBuf, work_dir: PathBuf, last_assigned_port: u16, - storage_cplane: &'a StorageControlPlane, + pageserver: Arc, nodes: Vec>, + env: LocalEnv, } -impl ComputeControlPlane<'_> { - pub fn local(storage_cplane: &StorageControlPlane) -> ComputeControlPlane { +impl ComputeControlPlane { + pub fn local(pageserver: &Arc) -> ComputeControlPlane { + let env = local_env::test_env(); ComputeControlPlane { - pg_bin_dir: PG_BIN_DIR.to_path_buf(), - work_dir: TEST_WORKDIR.to_path_buf(), + pg_bin_dir: env.pg_bin_dir(), + work_dir: env.data_dir.clone(), last_assigned_port: 65431, - storage_cplane: storage_cplane, + pageserver: Arc::clone(pageserver), nodes: Vec::new(), + env: env.clone(), } } @@ -296,24 +308,29 @@ impl ComputeControlPlane<'_> { let node_id = self.nodes.len() + 1; let node = PostgresNode { _node_id: node_id, - port: self.get_port(), - ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), - pg_bin_dir: self.pg_bin_dir.clone(), + env: self.env.clone(), + pageserver: Arc::clone(&self.pageserver), }; self.nodes.push(Arc::new(node)); let node = self.nodes.last().unwrap(); + println!( + "Creating new postgres: path={} port={}", + node.pgdata.to_str().unwrap(), + node.address.port() + ); + // initialize data directory fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok(); let initdb_path = self.pg_bin_dir.join("initdb"); - println!("initdb_path: {}", initdb_path.to_str().unwrap()); let initdb = Command::new(initdb_path) .args(&["-D", node.pgdata.to_str().unwrap()]) .arg("-N") .arg("--no-instructions") .env_clear() - .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap()) + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) .status() .expect("failed to execute initdb"); @@ -340,8 +357,8 @@ impl ComputeControlPlane<'_> { listen_addresses = '{address}'\n\ port = {port}\n\ ", - address = node.ip, - port = node.port + address = node.address.ip(), + port = node.address.port() ) .as_str(), ); @@ -357,10 +374,10 @@ impl ComputeControlPlane<'_> { let node_id = self.nodes.len() + 1; let node = PostgresNode { _node_id: node_id, - port: self.get_port(), - ip: IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), - pg_bin_dir: self.pg_bin_dir.clone(), + env: self.env.clone(), + pageserver: Arc::clone(&self.pageserver), }; self.nodes.push(Arc::new(node)); let node = self.nodes.last().unwrap(); @@ -375,7 +392,7 @@ impl ComputeControlPlane<'_> { .arg("--no-instructions") .arg("--compute-node") .env_clear() - .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap()) + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) .status() .expect("failed to execute initdb"); @@ -398,8 +415,8 @@ impl ComputeControlPlane<'_> { port = {port}\n\ computenode_mode = true\n\ ", - address = node.ip, - port = node.port + address = node.address.ip(), + port = node.address.port() ) .as_str(), ); @@ -408,20 +425,18 @@ impl ComputeControlPlane<'_> { } pub fn new_node(&mut self) -> Arc { - let storage_cplane = self.storage_cplane; + let addr = self.pageserver.address().clone(); let node = self.new_vanilla_node(); - let pserver = storage_cplane.page_server_addr(); - // Configure that node to take pages from pageserver node.append_conf( "postgresql.conf", format!( "\ - page_server_connstring = 'host={} port={}'\n\ - ", - pserver.ip(), - pserver.port() + page_server_connstring = 'host={} port={}'\n\ + ", + addr.ip(), + addr.port() ) .as_str(), ); @@ -434,8 +449,7 @@ impl ComputeControlPlane<'_> { node.append_conf( "postgresql.conf", - "synchronous_standby_names = 'safekeeper_proxy'\n\ - ", + "synchronous_standby_names = 'safekeeper_proxy'\n", ); node.clone() } @@ -470,11 +484,11 @@ impl Drop for WalProposerNode { /////////////////////////////////////////////////////////////////////////////// pub struct PostgresNode { + pub address: SocketAddr, _node_id: usize, - pub port: u16, - pub ip: IpAddr, pgdata: PathBuf, - pg_bin_dir: PathBuf, + pub env: LocalEnv, + pageserver: Arc, } impl PostgresNode { @@ -488,7 +502,7 @@ impl PostgresNode { } fn pg_ctl(&self, args: &[&str], check_ok: bool) { - let pg_ctl_path = self.pg_bin_dir.join("pg_ctl"); + let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl"); let pg_ctl = Command::new(pg_ctl_path) .args( [ @@ -503,7 +517,7 @@ impl PostgresNode { .concat(), ) .env_clear() - .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap()) + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) .status() .expect("failed to execute pg_ctl"); @@ -512,11 +526,10 @@ impl PostgresNode { } } - pub fn start(&self, storage_cplane: &StorageControlPlane) { - if storage_cplane.page_servers.len() != 0 { - let _res = - storage_cplane.page_server_psql(format!("callmemaybe {}", self.connstr()).as_str()); - } + pub fn start(&self) { + let _res = self + .pageserver + .page_server_psql(format!("callmemaybe {}", self.connstr()).as_str()); println!("Starting postgres node at '{}'", self.connstr()); self.pg_ctl(&["start"], true); } @@ -530,7 +543,12 @@ impl PostgresNode { } pub fn connstr(&self) -> String { - format!("host={} port={} user={}", self.ip, self.port, self.whoami()) + format!( + "host={} port={} user={}", + self.address.ip(), + self.address.port(), + self.whoami() + ) } // XXX: cache that in control plane @@ -549,8 +567,8 @@ impl PostgresNode { pub fn safe_psql(&self, db: &str, sql: &str) -> Vec { let connstring = format!( "host={} port={} dbname={} user={}", - self.ip, - self.port, + self.address.ip(), + self.address.port(), db, self.whoami() ); @@ -563,8 +581,8 @@ impl PostgresNode { pub fn open_psql(&self, db: &str) -> Client { let connstring = format!( "host={} port={} dbname={} user={}", - self.ip, - self.port, + self.address.ip(), + self.address.port(), db, self.whoami() ); @@ -583,7 +601,7 @@ impl PostgresNode { File::create(filepath).unwrap(); } - let pg_resetwal_path = self.pg_bin_dir.join("pg_resetwal"); + let pg_resetwal_path = self.env.pg_bin_dir().join("pg_resetwal"); let pg_resetwal = Command::new(pg_resetwal_path) .args(&["-D", self.pgdata.to_str().unwrap()]) @@ -599,13 +617,13 @@ impl PostgresNode { } pub fn start_proxy(&self, wal_acceptors: String) -> WalProposerNode { - let proxy_path = PG_BIN_DIR.join("safekeeper_proxy"); + let proxy_path = self.env.pg_bin_dir().join("safekeeper_proxy"); match Command::new(proxy_path.as_path()) .args(&["-s", &wal_acceptors]) - .args(&["-h", &self.ip.to_string()]) - .args(&["-p", &self.port.to_string()]) + .args(&["-h", &self.address.ip().to_string()]) + .args(&["-p", &self.address.port().to_string()]) .arg("-v") - .stderr(File::create(TEST_WORKDIR.join("safepkeeper_proxy.log")).unwrap()) + .stderr(File::create(self.env.data_dir.join("safepkeeper_proxy.log")).unwrap()) .spawn() { Ok(child) => WalProposerNode { pid: child.id() }, @@ -644,7 +662,7 @@ pub fn regress_check(pg: &PostgresNode) { .args(&[ "--bindir=''", "--use-existing", - format!("--bindir={}", PG_BIN_DIR.to_str().unwrap()).as_str(), + format!("--bindir={}", pg.env.pg_bin_dir().to_str().unwrap()).as_str(), format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(), format!( "--schedule={}", @@ -654,10 +672,10 @@ pub fn regress_check(pg: &PostgresNode) { format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(), ]) .env_clear() - .env("LD_LIBRARY_PATH", PG_LIB_DIR.to_str().unwrap()) - .env("PGPORT", pg.port.to_string()) + .env("LD_LIBRARY_PATH", pg.env.pg_lib_dir().to_str().unwrap()) + .env("PGHOST", pg.address.ip().to_string()) + .env("PGPORT", pg.address.port().to_string()) .env("PGUSER", pg.whoami()) - .env("PGHOST", pg.ip.to_string()) .status() .expect("pg_regress failed"); } diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs new file mode 100644 index 0000000000..27f79a644c --- /dev/null +++ b/control_plane/src/local_env.rs @@ -0,0 +1,210 @@ +// +// This module is responsible for locating and loading paths in a local setup. +// +// Now it also provides init method which acts like a stub for proper installation +// script which will use local paths. +// +use std::env; +use std::error; +use std::fs; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; + +use home; +use serde_derive::{Deserialize, Serialize}; + +type Result = std::result::Result>; + +// +// This data structure represents deserialized zenith config, which should be +// located in ~/.zenith +// +// TODO: should we also support ZENITH_CONF env var? +// +#[derive(Serialize, Deserialize, Clone)] +pub struct LocalEnv { + // Here page server and compute nodes will create and store their data. + pub data_dir: PathBuf, + + // Path to postgres distribution. It expected that "bin", "include", + // "lib", "share" from postgres distribution will be there. If at some point + // in time we will be able to run against vanilla postgres we may split that + // to four separate paths and match OS-specific installation layout. + pub pg_distrib_dir: PathBuf, + + // Path to pageserver binary. + pub zenith_distrib_dir: PathBuf, + + // PageServer-specific options. + pub pageserver: PageServerConf, +} + +impl LocalEnv { + pub fn pg_bin_dir(&self) -> PathBuf { + self.pg_distrib_dir.join("bin") + } + pub fn pg_lib_dir(&self) -> PathBuf { + self.pg_distrib_dir.join("lib") + } + + pub fn pageserver_data_dir(&self) -> PathBuf { + self.data_dir.join("pageserver") + } + pub fn pageserver_log(&self) -> PathBuf { + self.pageserver_data_dir().join("pageserver.log") + } + pub fn pageserver_pidfile(&self) -> PathBuf { + self.pageserver_data_dir().join("pageserver.pid") + } +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct PageServerConf { + pub listen_address: SocketAddr, +} + +// +// Issues in rust-lang repo has several discussions about proper library to check +// home directory in a cross-platform way. Seems that current consensus is around +// home crate and cargo uses it. +// +fn get_home() -> Result { + match home::home_dir() { + Some(path) => Ok(path), + None => { + return Err(Box::::from( + "can not determine home directory path", + )); + } + } +} + +pub fn init() -> Result<()> { + let home_dir = get_home()?; + + // check if config already exists + let cfg_path = home_dir.join(".zenith"); + if cfg_path.exists() { + let err_msg = format!( + "{} already exists. Perhaps already initialized?", + cfg_path.to_str().unwrap() + ); + return Err(Box::::from(err_msg)); + } + + // Now we can run init only from crate directory, so check that current dir is our crate. + // Use 'pageserver/Cargo.toml' existence as evidendce. + let cargo_path = env::current_dir()?; + if !cargo_path.join("pageserver/Cargo.toml").exists() { + let err_msg = "Current dirrectory does not look like a zenith repo. \ + Please, run 'init' from zenith repo root."; + return Err(Box::::from(err_msg)); + } + + // ok, now check that expected binaries are present + + // check postgres + let pg_distrib_dir = cargo_path.join("tmp_install"); + let pg_path = pg_distrib_dir.join("bin/postgres"); + if !pg_path.exists() { + let err_msg = format!( + "Can't find postres binary at {}. \ + Perhaps './pgbuild.sh' is needed to build it first.", + pg_path.to_str().unwrap() + ); + return Err(Box::::from(err_msg)); + } + + // check pageserver + let zenith_distrib_dir = cargo_path.join("target/debug/"); + let pageserver_path = zenith_distrib_dir.join("pageserver"); + if !pageserver_path.exists() { + let err_msg = format!( + "Can't find pageserver binary at {}. Please build it.", + pageserver_path.to_str().unwrap() + ); + return Err(Box::::from(err_msg)); + } + + // ok, we are good to go + + // create data dir + let data_dir = cargo_path.join("tmp_install"); + match fs::create_dir(data_dir.clone()) { + Ok(_) => {} + Err(e) => match e.kind() { + std::io::ErrorKind::AlreadyExists => {} + _ => { + let err_msg = format!( + "Failed to create data directory in '{}': {}", + data_dir.to_str().unwrap(), + e + ); + return Err(Box::::from(err_msg)); + } + }, + } + + // write config + let conf = LocalEnv { + data_dir, + pg_distrib_dir, + zenith_distrib_dir, + pageserver: PageServerConf { + listen_address: "127.0.0.1:5430".parse().unwrap(), + }, + }; + let toml = toml::to_string(&conf)?; + fs::write(cfg_path, toml)?; + + Ok(()) +} + +// check that config file is present +pub fn load_config() -> Result { + // home + let home_dir = get_home()?; + + // check file exists + let cfg_path = home_dir.join(".zenith"); + if !cfg_path.exists() { + let err_msg = format!( + "Zenith config is not found in {}. You need to run 'zenith init' first", + cfg_path.to_str().unwrap() + ); + return Err(Box::::from(err_msg)); + } + + // load and parse file + let config = fs::read_to_string(cfg_path)?; + match toml::from_str(config.as_str()) { + Ok(cfg) => Ok(cfg), + Err(e) => Err(Box::::from(e)), + } +} + +// local env for tests +pub fn test_env() -> LocalEnv { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tmp_check"); + fs::create_dir_all(data_dir.clone()).unwrap(); + LocalEnv { + data_dir, + pg_distrib_dir: Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install"), + zenith_distrib_dir: cargo_bin_dir(), + pageserver: PageServerConf { + listen_address: "127.0.0.1:65200".parse().unwrap(), + }, + } +} + +// Find the directory where the binaries were put (i.e. target/debug/) +pub fn cargo_bin_dir() -> PathBuf { + let mut pathbuf = std::env::current_exe().ok().unwrap(); + + pathbuf.pop(); + if pathbuf.ends_with("deps") { + pathbuf.pop(); + } + + return pathbuf; +} diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore new file mode 100644 index 0000000000..80006e4280 --- /dev/null +++ b/integration_tests/.gitignore @@ -0,0 +1 @@ +tmp_check/ \ No newline at end of file diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 6481bdbe29..ad7913cc5c 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -14,3 +14,4 @@ tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "r pageserver = { path = "../pageserver" } walkeeper = { path = "../walkeeper" } +control_plane = { path = "../control_plane" } diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index 8adacb3c54..824da68b1b 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -1,8 +1,7 @@ #[allow(dead_code)] -mod control_plane; - +// mod control_plane; use control_plane::ComputeControlPlane; -use control_plane::StorageControlPlane; +use control_plane::TestStorageControlPlane; // XXX: force all redo at the end // -- restart + seqscan won't read deleted stuff @@ -12,12 +11,12 @@ use control_plane::StorageControlPlane; #[test] fn test_redo_cases() { // Start pageserver that reads WAL directly from that postgres - let storage_cplane = StorageControlPlane::one_page_server(); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + let storage_cplane = TestStorageControlPlane::one_page_server(); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // start postgres let node = compute_cplane.new_node(); - node.start(&storage_cplane); + node.start(); // check basic work with table node.safe_psql( @@ -49,14 +48,15 @@ fn test_redo_cases() { // Runs pg_regress on a compute node #[test] +#[ignore] fn test_regress() { // Start pageserver that reads WAL directly from that postgres - let storage_cplane = StorageControlPlane::one_page_server(); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + let storage_cplane = TestStorageControlPlane::one_page_server(); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // start postgres let node = compute_cplane.new_node(); - node.start(&storage_cplane); + node.start(); control_plane::regress_check(&node); } @@ -65,14 +65,14 @@ fn test_regress() { #[test] fn test_pageserver_multitenancy() { // Start pageserver that reads WAL directly from that postgres - let storage_cplane = StorageControlPlane::one_page_server(); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + let storage_cplane = TestStorageControlPlane::one_page_server(); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // Allocate postgres instance, but don't start let node1 = compute_cplane.new_node(); let node2 = compute_cplane.new_node(); - node1.start(&storage_cplane); - node2.start(&storage_cplane); + node1.start(); + node2.start(); // check node1 node1.safe_psql( diff --git a/integration_tests/tests/test_wal_acceptor.rs b/integration_tests/tests/test_wal_acceptor.rs index df0d55413d..fbf474f27f 100644 --- a/integration_tests/tests/test_wal_acceptor.rs +++ b/integration_tests/tests/test_wal_acceptor.rs @@ -1,8 +1,6 @@ // Restart acceptors one by one while compute is under the load. -#[allow(dead_code)] -mod control_plane; use control_plane::ComputeControlPlane; -use control_plane::StorageControlPlane; +use control_plane::TestStorageControlPlane; use rand::Rng; use std::sync::Arc; @@ -13,13 +11,13 @@ use std::{thread, time}; fn test_acceptors_normal_work() { // Start pageserver that reads WAL directly from that postgres const REDUNDANCY: usize = 3; - let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + let storage_cplane = TestStorageControlPlane::fault_tolerant(REDUNDANCY); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); - // start postgre + // start postgres let node = compute_cplane.new_master_node(); - node.start(&storage_cplane); + node.start(); // start proxy let _proxy = node.start_proxy(wal_acceptors); @@ -50,14 +48,14 @@ fn test_acceptors_restarts() { const REDUNDANCY: usize = 3; const FAULT_PROBABILITY: f32 = 0.01; - let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + let storage_cplane = TestStorageControlPlane::fault_tolerant(REDUNDANCY); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); let mut rng = rand::thread_rng(); - // start postgre + // start postgres let node = compute_cplane.new_master_node(); - node.start(&storage_cplane); + node.start(); // start proxy let _proxy = node.start_proxy(wal_acceptors); @@ -93,7 +91,7 @@ fn test_acceptors_restarts() { assert_eq!(count, 500500); } -fn start_acceptor(cplane: &Arc, no: usize) { +fn start_acceptor(cplane: &Arc, no: usize) { let cp = cplane.clone(); thread::spawn(move || { thread::sleep(time::Duration::from_secs(1)); @@ -109,13 +107,13 @@ fn test_acceptors_unavalability() { // Start pageserver that reads WAL directly from that postgres const REDUNDANCY: usize = 2; - let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + let storage_cplane = TestStorageControlPlane::fault_tolerant(REDUNDANCY); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); - // start postgre + // start postgres let node = compute_cplane.new_master_node(); - node.start(&storage_cplane); + node.start(); // start proxy let _proxy = node.start_proxy(wal_acceptors); @@ -157,11 +155,11 @@ fn test_acceptors_unavalability() { assert_eq!(count, 15); } -fn simulate_failures(cplane: &Arc) { +fn simulate_failures(cplane: Arc) { let mut rng = rand::thread_rng(); let n_acceptors = cplane.wal_acceptors.len(); let failure_period = time::Duration::from_secs(1); - loop { + while cplane.is_running() { thread::sleep(failure_period); let mask: u32 = rng.gen_range(0..(1 << n_acceptors)); for i in 0..n_acceptors { @@ -184,13 +182,13 @@ fn test_race_conditions() { // Start pageserver that reads WAL directly from that postgres const REDUNDANCY: usize = 3; - let storage_cplane = StorageControlPlane::fault_tolerant(REDUNDANCY); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + let storage_cplane = Arc::new(TestStorageControlPlane::fault_tolerant(REDUNDANCY)); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); - // start postgre + // start postgres let node = compute_cplane.new_master_node(); - node.start(&storage_cplane); + node.start(); // start proxy let _proxy = node.start_proxy(wal_acceptors); @@ -200,10 +198,10 @@ fn test_race_conditions() { "postgres", "CREATE TABLE t(key int primary key, value text)", ); - let cplane = Arc::new(storage_cplane); - let cp = cplane.clone(); - thread::spawn(move || { - simulate_failures(&cp); + + let cp = storage_cplane.clone(); + let failures_thread = thread::spawn(move || { + simulate_failures(cp); }); let mut psql = node.open_psql("postgres"); @@ -218,5 +216,7 @@ fn test_race_conditions() { .get(0); println!("sum = {}", count); assert_eq!(count, 500500); - cplane.stop(); + + storage_cplane.stop(); + failures_thread.join().unwrap(); } diff --git a/zenith/Cargo.toml b/zenith/Cargo.toml new file mode 100644 index 0000000000..2d1f7c922c --- /dev/null +++ b/zenith/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "zenith" +version = "0.1.0" +authors = ["Stas Kelvich "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = "2.33.0" +control_plane = { path = "../control_plane" } diff --git a/zenith/src/main.rs b/zenith/src/main.rs new file mode 100644 index 0000000000..20b48ba5f1 --- /dev/null +++ b/zenith/src/main.rs @@ -0,0 +1,119 @@ +use clap::{App, SubCommand}; +use std::fs; +use std::process::exit; +use std::process::Command; + +use control_plane::local_env; + +fn main() { + let matches = App::new("zenith") + .subcommand(SubCommand::with_name("init")) + .subcommand(SubCommand::with_name("start")) + .subcommand(SubCommand::with_name("stop")) + .subcommand(SubCommand::with_name("status")) + .subcommand( + SubCommand::with_name("pg") + .about("Manage postgres instances") + .subcommand(SubCommand::with_name("create")) + .subcommand(SubCommand::with_name("start")) + .subcommand(SubCommand::with_name("stop")) + .subcommand(SubCommand::with_name("destroy")), + ) + .subcommand( + SubCommand::with_name("snapshot") + .about("Manage database snapshots") + .subcommand(SubCommand::with_name("create")) + .subcommand(SubCommand::with_name("start")) + .subcommand(SubCommand::with_name("stop")) + .subcommand(SubCommand::with_name("destroy")), + ) + .get_matches(); + + // handle init separately and exit + if let Some("init") = matches.subcommand_name() { + match local_env::init() { + Ok(_) => { + println!("Initialization complete! You may start zenith with 'zenith start' now."); + exit(0); + } + Err(e) => { + eprintln!("Error during init: {}", e); + exit(1); + } + } + } + + // all other commands would need config + let conf = match local_env::load_config() { + Ok(conf) => conf, + Err(e) => { + eprintln!("Error loading config from ~/.zenith: {}", e); + exit(1); + } + }; + + match matches.subcommand() { + ("init", Some(_)) => { + panic!() /* init was handled before */ + } + + ("start", Some(_sub_m)) => { + println!( + "Starting pageserver at '{}'", + conf.pageserver.listen_address + ); + + let status = Command::new(conf.zenith_distrib_dir.join("pageserver")) + .args(&["-D", conf.data_dir.to_str().unwrap()]) + .args(&["-l", conf.pageserver.listen_address.to_string().as_str()]) + .arg("-d") + .arg("--skip-recovery") + .env_clear() + .env("PATH", conf.pg_bin_dir().to_str().unwrap()) // pageserver needs postres-wal-redo binary + .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap()) + .status() + .expect("failed to start pageserver"); + + if !status.success() { + eprintln!( + "Pageserver failed to start. See '{}' for details.", + conf.pageserver_log().to_str().unwrap() + ); + exit(1); + } + + // TODO: check it's actually started, or run status + + println!("Done!"); + } + + ("stop", Some(_sub_m)) => { + let pid = fs::read_to_string(conf.pageserver_pidfile()).unwrap(); + let status = Command::new("kill") + .arg(pid) + .env_clear() + .status() + .expect("failed to execute kill"); + + if !status.success() { + eprintln!("Failed to kill pageserver"); + exit(1); + } + + println!("Done!"); + } + + ("status", Some(_sub_m)) => {} + + ("pg", Some(pg_match)) => { + match pg_match.subcommand() { + ("start", Some(_sub_m)) => { + println!("xxx: pg start"); + // Ok(()) + } + _ => {} + } + } + _ => {} + } +} From 6264dc6aa3c72070a51e59780a434094b2f8076c Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Sat, 10 Apr 2021 13:56:19 +0300 Subject: [PATCH 02/33] Move control_plane code out of lib.rs and split up control plane into compute and storage parts. Before that code was concentrated in lib.rs which was unhandy to open by name. --- control_plane/src/compute.rs | 363 ++++++++++ control_plane/src/lib.rs | 673 +------------------ control_plane/src/storage.rs | 319 +++++++++ integration_tests/tests/test_pageserver.rs | 6 +- integration_tests/tests/test_wal_acceptor.rs | 4 +- 5 files changed, 689 insertions(+), 676 deletions(-) create mode 100644 control_plane/src/compute.rs create mode 100644 control_plane/src/storage.rs diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs new file mode 100644 index 0000000000..bb7a177893 --- /dev/null +++ b/control_plane/src/compute.rs @@ -0,0 +1,363 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::fs::{self, OpenOptions}; +use std::process::Command; +use std::fs::File; +use std::{ + io::Write, + net::{IpAddr, Ipv4Addr, SocketAddr}, +}; + +use postgres::{Client, NoTls}; + +use crate::local_env::{self, LocalEnv}; +use crate::storage::{PageServerNode, WalProposerNode}; + +// +// ComputeControlPlane +// +pub struct ComputeControlPlane { + pg_bin_dir: PathBuf, + work_dir: PathBuf, + last_assigned_port: u16, + pageserver: Arc, + nodes: Vec>, + env: LocalEnv, +} + +impl ComputeControlPlane { + pub fn local(pageserver: &Arc) -> ComputeControlPlane { + let env = local_env::test_env(); + ComputeControlPlane { + pg_bin_dir: env.pg_bin_dir(), + work_dir: env.data_dir.clone(), + last_assigned_port: 65431, + pageserver: Arc::clone(pageserver), + nodes: Vec::new(), + env: env.clone(), + } + } + + // TODO: check port availability and + fn get_port(&mut self) -> u16 { + let port = self.last_assigned_port + 1; + self.last_assigned_port += 1; + port + } + + pub fn new_vanilla_node<'a>(&mut self) -> &Arc { + // allocate new node entry with generated port + let node_id = self.nodes.len() + 1; + let node = PostgresNode { + _node_id: node_id, + address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), + pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), + env: self.env.clone(), + pageserver: Arc::clone(&self.pageserver), + }; + self.nodes.push(Arc::new(node)); + let node = self.nodes.last().unwrap(); + + println!( + "Creating new postgres: path={} port={}", + node.pgdata.to_str().unwrap(), + node.address.port() + ); + + // initialize data directory + fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok(); + let initdb_path = self.pg_bin_dir.join("initdb"); + let initdb = Command::new(initdb_path) + .args(&["-D", node.pgdata.to_str().unwrap()]) + .arg("-N") + .arg("--no-instructions") + .env_clear() + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) + .status() + .expect("failed to execute initdb"); + + if !initdb.success() { + panic!("initdb failed"); + } + + // // allow local replication connections + // node.append_conf("pg_hba.conf", format!("\ + // host replication all {}/32 sspi include_realm=1 map=regress\n\ + // ", node.ip).as_str()); + + // listen for selected port + node.append_conf( + "postgresql.conf", + format!( + "\ + max_wal_senders = 10\n\ + max_replication_slots = 10\n\ + hot_standby = on\n\ + shared_buffers = 1MB\n\ + max_connections = 100\n\ + wal_level = replica\n\ + listen_addresses = '{address}'\n\ + port = {port}\n\ + ", + address = node.address.ip(), + port = node.address.port() + ) + .as_str(), + ); + + node + } + + // Init compute node without files, only datadir structure + // use initdb --compute-node flag and GUC 'computenode_mode' + // to distinguish the node + pub fn new_minimal_node(&mut self) -> &PostgresNode { + // allocate new node entry with generated port + let node_id = self.nodes.len() + 1; + let node = PostgresNode { + _node_id: node_id, + address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), + pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), + env: self.env.clone(), + pageserver: Arc::clone(&self.pageserver), + }; + self.nodes.push(Arc::new(node)); + let node = self.nodes.last().unwrap(); + + // initialize data directory w/o files + fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok(); + let initdb_path = self.pg_bin_dir.join("initdb"); + println!("initdb_path: {}", initdb_path.to_str().unwrap()); + let initdb = Command::new(initdb_path) + .args(&["-D", node.pgdata.to_str().unwrap()]) + .arg("-N") + .arg("--no-instructions") + .arg("--compute-node") + .env_clear() + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) + .status() + .expect("failed to execute initdb"); + + if !initdb.success() { + panic!("initdb failed"); + } + + // listen for selected port + node.append_conf( + "postgresql.conf", + format!( + "\ + max_wal_senders = 10\n\ + max_replication_slots = 10\n\ + hot_standby = on\n\ + shared_buffers = 1MB\n\ + max_connections = 100\n\ + wal_level = replica\n\ + listen_addresses = '{address}'\n\ + port = {port}\n\ + computenode_mode = true\n\ + ", + address = node.address.ip(), + port = node.address.port() + ) + .as_str(), + ); + + node + } + + pub fn new_node(&mut self) -> Arc { + let addr = self.pageserver.address().clone(); + let node = self.new_vanilla_node(); + + // Configure that node to take pages from pageserver + node.append_conf( + "postgresql.conf", + format!( + "\ + page_server_connstring = 'host={} port={}'\n\ + ", + addr.ip(), + addr.port() + ) + .as_str(), + ); + + node.clone() + } + + pub fn new_master_node(&mut self) -> Arc { + let node = self.new_vanilla_node(); + + node.append_conf( + "postgresql.conf", + "synchronous_standby_names = 'safekeeper_proxy'\n", + ); + node.clone() + } +} + +/////////////////////////////////////////////////////////////////////////////// + +pub struct PostgresNode { + pub address: SocketAddr, + _node_id: usize, + pgdata: PathBuf, + pub env: LocalEnv, + pageserver: Arc, +} + +impl PostgresNode { + pub fn append_conf(&self, config: &str, opts: &str) { + OpenOptions::new() + .append(true) + .open(self.pgdata.join(config).to_str().unwrap()) + .unwrap() + .write_all(opts.as_bytes()) + .unwrap(); + } + + fn pg_ctl(&self, args: &[&str], check_ok: bool) { + let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl"); + let pg_ctl = Command::new(pg_ctl_path) + .args( + [ + &[ + "-D", + self.pgdata.to_str().unwrap(), + "-l", + self.pgdata.join("log").to_str().unwrap(), + ], + args, + ] + .concat(), + ) + .env_clear() + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) + .status() + .expect("failed to execute pg_ctl"); + + if check_ok && !pg_ctl.success() { + panic!("pg_ctl failed"); + } + } + + pub fn start(&self) { + let _res = self + .pageserver + .page_server_psql(format!("callmemaybe {}", self.connstr()).as_str()); + println!("Starting postgres node at '{}'", self.connstr()); + self.pg_ctl(&["start"], true); + } + + pub fn restart(&self) { + self.pg_ctl(&["restart"], true); + } + + pub fn stop(&self) { + self.pg_ctl(&["-m", "immediate", "stop"], true); + } + + pub fn connstr(&self) -> String { + format!( + "host={} port={} user={}", + self.address.ip(), + self.address.port(), + self.whoami() + ) + } + + // XXX: cache that in control plane + pub fn whoami(&self) -> String { + let output = Command::new("whoami") + .output() + .expect("failed to execute whoami"); + + if !output.status.success() { + panic!("whoami failed"); + } + + String::from_utf8(output.stdout).unwrap().trim().to_string() + } + + pub fn safe_psql(&self, db: &str, sql: &str) -> Vec { + let connstring = format!( + "host={} port={} dbname={} user={}", + self.address.ip(), + self.address.port(), + db, + self.whoami() + ); + let mut client = Client::connect(connstring.as_str(), NoTls).unwrap(); + + println!("Running {}", sql); + client.query(sql, &[]).unwrap() + } + + pub fn open_psql(&self, db: &str) -> Client { + let connstring = format!( + "host={} port={} dbname={} user={}", + self.address.ip(), + self.address.port(), + db, + self.whoami() + ); + Client::connect(connstring.as_str(), NoTls).unwrap() + } + + pub fn get_pgdata(&self) -> Option<&str> { + self.pgdata.to_str() + } + + /* Create stub controlfile and respective xlog to start computenode */ + pub fn setup_controlfile(&self) { + let filepath = format!("{}/global/pg_control", self.pgdata.to_str().unwrap()); + + { + File::create(filepath).unwrap(); + } + + let pg_resetwal_path = self.env.pg_bin_dir().join("pg_resetwal"); + + let pg_resetwal = Command::new(pg_resetwal_path) + .args(&["-D", self.pgdata.to_str().unwrap()]) + .arg("-f") + // TODO probably we will have to modify pg_resetwal + // .arg("--compute-node") + .status() + .expect("failed to execute pg_resetwal"); + + if !pg_resetwal.success() { + panic!("pg_resetwal failed"); + } + } + + pub fn start_proxy(&self, wal_acceptors: String) -> WalProposerNode { + let proxy_path = self.env.pg_bin_dir().join("safekeeper_proxy"); + match Command::new(proxy_path.as_path()) + .args(&["-s", &wal_acceptors]) + .args(&["-h", &self.address.ip().to_string()]) + .args(&["-p", &self.address.port().to_string()]) + .arg("-v") + .stderr(File::create(self.env.data_dir.join("safepkeeper_proxy.log")).unwrap()) + .spawn() + { + Ok(child) => WalProposerNode { pid: child.id() }, + Err(e) => panic!("Failed to launch {:?}: {}", proxy_path, e), + } + } + + // TODO + pub fn pg_bench() {} + pub fn pg_regress() {} +} + +impl Drop for PostgresNode { + // destructor to clean up state after test is done + // XXX: we may detect failed test by setting some flag in catch_unwind() + // and checking it here. But let just clean datadirs on start. + fn drop(&mut self) { + self.stop(); + // fs::remove_dir_all(self.pgdata.clone()).unwrap(); + } +} diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 09e172ec4a..2c2857cb64 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -7,675 +7,6 @@ // local installations. // -use std::fs::File; -use std::fs::{self, OpenOptions}; -use std::net::TcpStream; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; -use std::thread; -use std::time::Duration; -use std::{ - io::Write, - net::{IpAddr, Ipv4Addr, SocketAddr}, -}; - pub mod local_env; -use local_env::LocalEnv; -use postgres::{Client, NoTls}; - -// -// Collection of several example deployments useful for tests. -// -// I'm intendedly modelling storage and compute control planes as a separate entities -// as it is closer to the actual setup. -// -pub struct TestStorageControlPlane { - pub wal_acceptors: Vec, - pub pageserver: Arc, - pub test_done: AtomicBool, -} - -impl TestStorageControlPlane { - // postgres <-> page_server - pub fn one_page_server() -> TestStorageControlPlane { - let env = local_env::test_env(); - - let pserver = Arc::new(PageServerNode { - env: env.clone(), - kill_on_exit: true, - }); - pserver.init(); - pserver.start(); - - TestStorageControlPlane { - wal_acceptors: Vec::new(), - pageserver: pserver, - test_done: AtomicBool::new(false), - } - } - - // postgres <-> {wal_acceptor1, wal_acceptor2, ...} - pub fn fault_tolerant(redundancy: usize) -> TestStorageControlPlane { - let env = local_env::test_env(); - let mut cplane = TestStorageControlPlane { - wal_acceptors: Vec::new(), - pageserver: Arc::new(PageServerNode { - env: env.clone(), - kill_on_exit: true, - }), - test_done: AtomicBool::new(false), - }; - cplane.pageserver.init(); - cplane.pageserver.start(); - - const WAL_ACCEPTOR_PORT: usize = 54321; - - for i in 0..redundancy { - let wal_acceptor = WalAcceptorNode { - listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i) - .parse() - .unwrap(), - data_dir: env.data_dir.join(format!("wal_acceptor_{}", i)), - env: env.clone(), - }; - wal_acceptor.init(); - wal_acceptor.start(); - cplane.wal_acceptors.push(wal_acceptor); - } - cplane - } - - pub fn stop(&self) { - self.test_done.store(true, Ordering::Relaxed); - } - - pub fn get_wal_acceptor_conn_info(&self) -> String { - self.wal_acceptors - .iter() - .map(|wa| wa.listen.to_string().to_string()) - .collect::>() - .join(",") - } - - pub fn is_running(&self) -> bool { - self.test_done.load(Ordering::Relaxed) - } -} - -impl Drop for TestStorageControlPlane { - fn drop(&mut self) { - self.stop(); - } -} - -// -// Control routines for pageserver. -// -// Used in CLI and tests. -// -pub struct PageServerNode { - kill_on_exit: bool, - env: LocalEnv, -} - -impl PageServerNode { - pub fn init(&self) { - fs::create_dir_all(self.env.pageserver_data_dir()).unwrap(); - } - - pub fn start(&self) { - println!( - "Starting pageserver at '{}'", - self.env.pageserver.listen_address - ); - - let status = Command::new(self.env.zenith_distrib_dir.join("pageserver")) // XXX -> method - .args(&["-D", self.env.pageserver_data_dir().to_str().unwrap()]) - .args(&[ - "-l", - self.env.pageserver.listen_address.to_string().as_str(), - ]) - .arg("-d") - .arg("--skip-recovery") - .env_clear() - .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary - .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to start pageserver"); - - if !status.success() { - panic!("pageserver start failed"); - } - } - - pub fn stop(&self) { - let pidfile = self.env.pageserver_pidfile(); - let pid = fs::read_to_string(pidfile).unwrap(); - - let status = Command::new("kill") - .arg(pid) - .env_clear() - .status() - .expect("failed to execute kill"); - - if !status.success() { - panic!("kill start failed"); - } - - // await for pageserver stop - for _ in 0..5 { - let stream = TcpStream::connect(self.env.pageserver.listen_address); - if let Err(_e) = stream { - return; - } - println!( - "Stopping pageserver on {}", - self.env.pageserver.listen_address - ); - thread::sleep(Duration::from_secs(1)); - } - - // ok, we failed to stop pageserver, let's panic - panic!("Failed to stop pageserver"); - } - - pub fn address(&self) -> &std::net::SocketAddr { - &self.env.pageserver.listen_address - } - - pub fn page_server_psql(&self, sql: &str) -> Vec { - // let addr = &self.page_servers[0].env.pageserver.listen_address; - - let connstring = format!( - "host={} port={} dbname={} user={}", - self.address().ip(), - self.address().port(), - "no_db", - "no_user", - ); - let mut client = Client::connect(connstring.as_str(), NoTls).unwrap(); - - println!("Pageserver query: '{}'", sql); - client.simple_query(sql).unwrap() - } -} - -impl Drop for PageServerNode { - fn drop(&mut self) { - if self.kill_on_exit { - self.stop(); - } - } -} - -// -// Control routines for WalAcceptor. -// -// Now used only in test setups. -// -pub struct WalAcceptorNode { - listen: SocketAddr, - data_dir: PathBuf, - env: LocalEnv, -} - -impl WalAcceptorNode { - pub fn init(&self) { - if self.data_dir.exists() { - fs::remove_dir_all(self.data_dir.clone()).unwrap(); - } - fs::create_dir_all(self.data_dir.clone()).unwrap(); - } - - pub fn start(&self) { - println!( - "Starting wal_acceptor in {} listening '{}'", - self.data_dir.to_str().unwrap(), - self.listen - ); - - let status = Command::new(self.env.zenith_distrib_dir.join("wal_acceptor")) - .args(&["-D", self.data_dir.to_str().unwrap()]) - .args(&["-l", self.listen.to_string().as_str()]) - .arg("-d") - .arg("-n") - .status() - .expect("failed to start wal_acceptor"); - - if !status.success() { - panic!("wal_acceptor start failed"); - } - } - - pub fn stop(&self) { - println!("Stopping wal acceptor on {}", self.listen); - let pidfile = self.data_dir.join("wal_acceptor.pid"); - if let Ok(pid) = fs::read_to_string(pidfile) { - let _status = Command::new("kill") - .arg(pid) - .env_clear() - .status() - .expect("failed to execute kill"); - } - } -} - -impl Drop for WalAcceptorNode { - fn drop(&mut self) { - self.stop(); - } -} - -/////////////////////////////////////////////////////////////////////////////// - -// -// ComputeControlPlane -// -pub struct ComputeControlPlane { - pg_bin_dir: PathBuf, - work_dir: PathBuf, - last_assigned_port: u16, - pageserver: Arc, - nodes: Vec>, - env: LocalEnv, -} - -impl ComputeControlPlane { - pub fn local(pageserver: &Arc) -> ComputeControlPlane { - let env = local_env::test_env(); - ComputeControlPlane { - pg_bin_dir: env.pg_bin_dir(), - work_dir: env.data_dir.clone(), - last_assigned_port: 65431, - pageserver: Arc::clone(pageserver), - nodes: Vec::new(), - env: env.clone(), - } - } - - // TODO: check port availability and - fn get_port(&mut self) -> u16 { - let port = self.last_assigned_port + 1; - self.last_assigned_port += 1; - port - } - - pub fn new_vanilla_node<'a>(&mut self) -> &Arc { - // allocate new node entry with generated port - let node_id = self.nodes.len() + 1; - let node = PostgresNode { - _node_id: node_id, - address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), - pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), - env: self.env.clone(), - pageserver: Arc::clone(&self.pageserver), - }; - self.nodes.push(Arc::new(node)); - let node = self.nodes.last().unwrap(); - - println!( - "Creating new postgres: path={} port={}", - node.pgdata.to_str().unwrap(), - node.address.port() - ); - - // initialize data directory - fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok(); - let initdb_path = self.pg_bin_dir.join("initdb"); - let initdb = Command::new(initdb_path) - .args(&["-D", node.pgdata.to_str().unwrap()]) - .arg("-N") - .arg("--no-instructions") - .env_clear() - .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to execute initdb"); - - if !initdb.success() { - panic!("initdb failed"); - } - - // // allow local replication connections - // node.append_conf("pg_hba.conf", format!("\ - // host replication all {}/32 sspi include_realm=1 map=regress\n\ - // ", node.ip).as_str()); - - // listen for selected port - node.append_conf( - "postgresql.conf", - format!( - "\ - max_wal_senders = 10\n\ - max_replication_slots = 10\n\ - hot_standby = on\n\ - shared_buffers = 1MB\n\ - max_connections = 100\n\ - wal_level = replica\n\ - listen_addresses = '{address}'\n\ - port = {port}\n\ - ", - address = node.address.ip(), - port = node.address.port() - ) - .as_str(), - ); - - node - } - - // Init compute node without files, only datadir structure - // use initdb --compute-node flag and GUC 'computenode_mode' - // to distinguish the node - pub fn new_minimal_node(&mut self) -> &PostgresNode { - // allocate new node entry with generated port - let node_id = self.nodes.len() + 1; - let node = PostgresNode { - _node_id: node_id, - address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), - pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), - env: self.env.clone(), - pageserver: Arc::clone(&self.pageserver), - }; - self.nodes.push(Arc::new(node)); - let node = self.nodes.last().unwrap(); - - // initialize data directory w/o files - fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok(); - let initdb_path = self.pg_bin_dir.join("initdb"); - println!("initdb_path: {}", initdb_path.to_str().unwrap()); - let initdb = Command::new(initdb_path) - .args(&["-D", node.pgdata.to_str().unwrap()]) - .arg("-N") - .arg("--no-instructions") - .arg("--compute-node") - .env_clear() - .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to execute initdb"); - - if !initdb.success() { - panic!("initdb failed"); - } - - // listen for selected port - node.append_conf( - "postgresql.conf", - format!( - "\ - max_wal_senders = 10\n\ - max_replication_slots = 10\n\ - hot_standby = on\n\ - shared_buffers = 1MB\n\ - max_connections = 100\n\ - wal_level = replica\n\ - listen_addresses = '{address}'\n\ - port = {port}\n\ - computenode_mode = true\n\ - ", - address = node.address.ip(), - port = node.address.port() - ) - .as_str(), - ); - - node - } - - pub fn new_node(&mut self) -> Arc { - let addr = self.pageserver.address().clone(); - let node = self.new_vanilla_node(); - - // Configure that node to take pages from pageserver - node.append_conf( - "postgresql.conf", - format!( - "\ - page_server_connstring = 'host={} port={}'\n\ - ", - addr.ip(), - addr.port() - ) - .as_str(), - ); - - node.clone() - } - - pub fn new_master_node(&mut self) -> Arc { - let node = self.new_vanilla_node(); - - node.append_conf( - "postgresql.conf", - "synchronous_standby_names = 'safekeeper_proxy'\n", - ); - node.clone() - } -} - -/////////////////////////////////////////////////////////////////////////////// - -pub struct WalProposerNode { - pid: u32, -} - -impl WalProposerNode { - pub fn stop(&self) { - let status = Command::new("kill") - .arg(self.pid.to_string()) - .env_clear() - .status() - .expect("failed to execute kill"); - - if !status.success() { - panic!("kill start failed"); - } - } -} - -impl Drop for WalProposerNode { - fn drop(&mut self) { - self.stop(); - } -} - -/////////////////////////////////////////////////////////////////////////////// - -pub struct PostgresNode { - pub address: SocketAddr, - _node_id: usize, - pgdata: PathBuf, - pub env: LocalEnv, - pageserver: Arc, -} - -impl PostgresNode { - pub fn append_conf(&self, config: &str, opts: &str) { - OpenOptions::new() - .append(true) - .open(self.pgdata.join(config).to_str().unwrap()) - .unwrap() - .write_all(opts.as_bytes()) - .unwrap(); - } - - fn pg_ctl(&self, args: &[&str], check_ok: bool) { - let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl"); - let pg_ctl = Command::new(pg_ctl_path) - .args( - [ - &[ - "-D", - self.pgdata.to_str().unwrap(), - "-l", - self.pgdata.join("log").to_str().unwrap(), - ], - args, - ] - .concat(), - ) - .env_clear() - .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to execute pg_ctl"); - - if check_ok && !pg_ctl.success() { - panic!("pg_ctl failed"); - } - } - - pub fn start(&self) { - let _res = self - .pageserver - .page_server_psql(format!("callmemaybe {}", self.connstr()).as_str()); - println!("Starting postgres node at '{}'", self.connstr()); - self.pg_ctl(&["start"], true); - } - - pub fn restart(&self) { - self.pg_ctl(&["restart"], true); - } - - pub fn stop(&self) { - self.pg_ctl(&["-m", "immediate", "stop"], true); - } - - pub fn connstr(&self) -> String { - format!( - "host={} port={} user={}", - self.address.ip(), - self.address.port(), - self.whoami() - ) - } - - // XXX: cache that in control plane - pub fn whoami(&self) -> String { - let output = Command::new("whoami") - .output() - .expect("failed to execute whoami"); - - if !output.status.success() { - panic!("whoami failed"); - } - - String::from_utf8(output.stdout).unwrap().trim().to_string() - } - - pub fn safe_psql(&self, db: &str, sql: &str) -> Vec { - let connstring = format!( - "host={} port={} dbname={} user={}", - self.address.ip(), - self.address.port(), - db, - self.whoami() - ); - let mut client = Client::connect(connstring.as_str(), NoTls).unwrap(); - - println!("Running {}", sql); - client.query(sql, &[]).unwrap() - } - - pub fn open_psql(&self, db: &str) -> Client { - let connstring = format!( - "host={} port={} dbname={} user={}", - self.address.ip(), - self.address.port(), - db, - self.whoami() - ); - Client::connect(connstring.as_str(), NoTls).unwrap() - } - - pub fn get_pgdata(&self) -> Option<&str> { - self.pgdata.to_str() - } - - /* Create stub controlfile and respective xlog to start computenode */ - pub fn setup_controlfile(&self) { - let filepath = format!("{}/global/pg_control", self.pgdata.to_str().unwrap()); - - { - File::create(filepath).unwrap(); - } - - let pg_resetwal_path = self.env.pg_bin_dir().join("pg_resetwal"); - - let pg_resetwal = Command::new(pg_resetwal_path) - .args(&["-D", self.pgdata.to_str().unwrap()]) - .arg("-f") - // TODO probably we will have to modify pg_resetwal - // .arg("--compute-node") - .status() - .expect("failed to execute pg_resetwal"); - - if !pg_resetwal.success() { - panic!("pg_resetwal failed"); - } - } - - pub fn start_proxy(&self, wal_acceptors: String) -> WalProposerNode { - let proxy_path = self.env.pg_bin_dir().join("safekeeper_proxy"); - match Command::new(proxy_path.as_path()) - .args(&["-s", &wal_acceptors]) - .args(&["-h", &self.address.ip().to_string()]) - .args(&["-p", &self.address.port().to_string()]) - .arg("-v") - .stderr(File::create(self.env.data_dir.join("safepkeeper_proxy.log")).unwrap()) - .spawn() - { - Ok(child) => WalProposerNode { pid: child.id() }, - Err(e) => panic!("Failed to launch {:?}: {}", proxy_path, e), - } - } - - // TODO - pub fn pg_bench() {} - pub fn pg_regress() {} -} - -impl Drop for PostgresNode { - // destructor to clean up state after test is done - // XXX: we may detect failed test by setting some flag in catch_unwind() - // and checking it here. But let just clean datadirs on start. - fn drop(&mut self) { - self.stop(); - // fs::remove_dir_all(self.pgdata.clone()).unwrap(); - } -} - -pub fn regress_check(pg: &PostgresNode) { - pg.safe_psql("postgres", "CREATE DATABASE regression"); - - let regress_run_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tmp_check/regress"); - fs::create_dir_all(regress_run_path.clone()).unwrap(); - std::env::set_current_dir(regress_run_path).unwrap(); - - let regress_build_path = - Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install/build/src/test/regress"); - let regress_src_path = - Path::new(env!("CARGO_MANIFEST_DIR")).join("../vendor/postgres/src/test/regress"); - - let _regress_check = Command::new(regress_build_path.join("pg_regress")) - .args(&[ - "--bindir=''", - "--use-existing", - format!("--bindir={}", pg.env.pg_bin_dir().to_str().unwrap()).as_str(), - format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(), - format!( - "--schedule={}", - regress_src_path.join("parallel_schedule").to_str().unwrap() - ) - .as_str(), - format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(), - ]) - .env_clear() - .env("LD_LIBRARY_PATH", pg.env.pg_lib_dir().to_str().unwrap()) - .env("PGHOST", pg.address.ip().to_string()) - .env("PGPORT", pg.address.port().to_string()) - .env("PGUSER", pg.whoami()) - .status() - .expect("pg_regress failed"); -} +pub mod compute; +pub mod storage; diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs new file mode 100644 index 0000000000..76c6257ae1 --- /dev/null +++ b/control_plane/src/storage.rs @@ -0,0 +1,319 @@ +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::fs; +use std::process::Command; +use std::net::TcpStream; +use std::thread; +use std::time::Duration; +use std::path::{Path, PathBuf}; +use std::net::SocketAddr; + +use postgres::{Client, NoTls}; + +use crate::local_env::{self, LocalEnv}; +use crate::compute::{PostgresNode}; + +// +// Collection of several example deployments useful for tests. +// +// I'm intendedly modelling storage and compute control planes as a separate entities +// as it is closer to the actual setup. +// +pub struct TestStorageControlPlane { + pub wal_acceptors: Vec, + pub pageserver: Arc, + pub test_done: AtomicBool, +} + +impl TestStorageControlPlane { + // postgres <-> page_server + pub fn one_page_server() -> TestStorageControlPlane { + let env = local_env::test_env(); + + let pserver = Arc::new(PageServerNode { + env: env.clone(), + kill_on_exit: true, + }); + pserver.init(); + pserver.start(); + + TestStorageControlPlane { + wal_acceptors: Vec::new(), + pageserver: pserver, + test_done: AtomicBool::new(false), + } + } + + // postgres <-> {wal_acceptor1, wal_acceptor2, ...} + pub fn fault_tolerant(redundancy: usize) -> TestStorageControlPlane { + let env = local_env::test_env(); + let mut cplane = TestStorageControlPlane { + wal_acceptors: Vec::new(), + pageserver: Arc::new(PageServerNode { + env: env.clone(), + kill_on_exit: true, + }), + test_done: AtomicBool::new(false), + }; + cplane.pageserver.init(); + cplane.pageserver.start(); + + const WAL_ACCEPTOR_PORT: usize = 54321; + + for i in 0..redundancy { + let wal_acceptor = WalAcceptorNode { + listen: format!("127.0.0.1:{}", WAL_ACCEPTOR_PORT + i) + .parse() + .unwrap(), + data_dir: env.data_dir.join(format!("wal_acceptor_{}", i)), + env: env.clone(), + }; + wal_acceptor.init(); + wal_acceptor.start(); + cplane.wal_acceptors.push(wal_acceptor); + } + cplane + } + + pub fn stop(&self) { + self.test_done.store(true, Ordering::Relaxed); + } + + pub fn get_wal_acceptor_conn_info(&self) -> String { + self.wal_acceptors + .iter() + .map(|wa| wa.listen.to_string().to_string()) + .collect::>() + .join(",") + } + + pub fn is_running(&self) -> bool { + self.test_done.load(Ordering::Relaxed) + } +} + +impl Drop for TestStorageControlPlane { + fn drop(&mut self) { + self.stop(); + } +} + +// +// Control routines for pageserver. +// +// Used in CLI and tests. +// +pub struct PageServerNode { + kill_on_exit: bool, + env: LocalEnv, +} + +impl PageServerNode { + pub fn init(&self) { + fs::create_dir_all(self.env.pageserver_data_dir()).unwrap(); + } + + pub fn start(&self) { + println!( + "Starting pageserver at '{}'", + self.env.pageserver.listen_address + ); + + let status = Command::new(self.env.zenith_distrib_dir.join("pageserver")) // XXX -> method + .args(&["-D", self.env.pageserver_data_dir().to_str().unwrap()]) + .args(&[ + "-l", + self.env.pageserver.listen_address.to_string().as_str(), + ]) + .arg("-d") + .arg("--skip-recovery") + .env_clear() + .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) + .status() + .expect("failed to start pageserver"); + + if !status.success() { + panic!("pageserver start failed"); + } + } + + pub fn stop(&self) { + let pidfile = self.env.pageserver_pidfile(); + let pid = fs::read_to_string(pidfile).unwrap(); + + let status = Command::new("kill") + .arg(pid) + .env_clear() + .status() + .expect("failed to execute kill"); + + if !status.success() { + panic!("kill start failed"); + } + + // await for pageserver stop + for _ in 0..5 { + let stream = TcpStream::connect(self.env.pageserver.listen_address); + if let Err(_e) = stream { + return; + } + println!( + "Stopping pageserver on {}", + self.env.pageserver.listen_address + ); + thread::sleep(Duration::from_secs(1)); + } + + // ok, we failed to stop pageserver, let's panic + panic!("Failed to stop pageserver"); + } + + pub fn address(&self) -> &std::net::SocketAddr { + &self.env.pageserver.listen_address + } + + pub fn page_server_psql(&self, sql: &str) -> Vec { + // let addr = &self.page_servers[0].env.pageserver.listen_address; + + let connstring = format!( + "host={} port={} dbname={} user={}", + self.address().ip(), + self.address().port(), + "no_db", + "no_user", + ); + let mut client = Client::connect(connstring.as_str(), NoTls).unwrap(); + + println!("Pageserver query: '{}'", sql); + client.simple_query(sql).unwrap() + } +} + +impl Drop for PageServerNode { + fn drop(&mut self) { + if self.kill_on_exit { + self.stop(); + } + } +} + +// +// Control routines for WalAcceptor. +// +// Now used only in test setups. +// +pub struct WalAcceptorNode { + listen: SocketAddr, + data_dir: PathBuf, + env: LocalEnv, +} + +impl WalAcceptorNode { + pub fn init(&self) { + if self.data_dir.exists() { + fs::remove_dir_all(self.data_dir.clone()).unwrap(); + } + fs::create_dir_all(self.data_dir.clone()).unwrap(); + } + + pub fn start(&self) { + println!( + "Starting wal_acceptor in {} listening '{}'", + self.data_dir.to_str().unwrap(), + self.listen + ); + + let status = Command::new(self.env.zenith_distrib_dir.join("wal_acceptor")) + .args(&["-D", self.data_dir.to_str().unwrap()]) + .args(&["-l", self.listen.to_string().as_str()]) + .arg("-d") + .arg("-n") + .status() + .expect("failed to start wal_acceptor"); + + if !status.success() { + panic!("wal_acceptor start failed"); + } + } + + pub fn stop(&self) { + println!("Stopping wal acceptor on {}", self.listen); + let pidfile = self.data_dir.join("wal_acceptor.pid"); + if let Ok(pid) = fs::read_to_string(pidfile) { + let _status = Command::new("kill") + .arg(pid) + .env_clear() + .status() + .expect("failed to execute kill"); + } + } +} + +impl Drop for WalAcceptorNode { + fn drop(&mut self) { + self.stop(); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +pub struct WalProposerNode { + pub pid: u32, +} + +impl WalProposerNode { + pub fn stop(&self) { + let status = Command::new("kill") + .arg(self.pid.to_string()) + .env_clear() + .status() + .expect("failed to execute kill"); + + if !status.success() { + panic!("kill start failed"); + } + } +} + +impl Drop for WalProposerNode { + fn drop(&mut self) { + self.stop(); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +pub fn regress_check(pg: &PostgresNode) { + pg.safe_psql("postgres", "CREATE DATABASE regression"); + + let regress_run_path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tmp_check/regress"); + fs::create_dir_all(regress_run_path.clone()).unwrap(); + std::env::set_current_dir(regress_run_path).unwrap(); + + let regress_build_path = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install/build/src/test/regress"); + let regress_src_path = + Path::new(env!("CARGO_MANIFEST_DIR")).join("../vendor/postgres/src/test/regress"); + + let _regress_check = Command::new(regress_build_path.join("pg_regress")) + .args(&[ + "--bindir=''", + "--use-existing", + format!("--bindir={}", pg.env.pg_bin_dir().to_str().unwrap()).as_str(), + format!("--dlpath={}", regress_build_path.to_str().unwrap()).as_str(), + format!( + "--schedule={}", + regress_src_path.join("parallel_schedule").to_str().unwrap() + ) + .as_str(), + format!("--inputdir={}", regress_src_path.to_str().unwrap()).as_str(), + ]) + .env_clear() + .env("LD_LIBRARY_PATH", pg.env.pg_lib_dir().to_str().unwrap()) + .env("PGHOST", pg.address.ip().to_string()) + .env("PGPORT", pg.address.port().to_string()) + .env("PGUSER", pg.whoami()) + .status() + .expect("pg_regress failed"); +} diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index 824da68b1b..2322f4d61e 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -1,7 +1,7 @@ #[allow(dead_code)] // mod control_plane; -use control_plane::ComputeControlPlane; -use control_plane::TestStorageControlPlane; +use control_plane::compute::ComputeControlPlane; +use control_plane::storage::TestStorageControlPlane; // XXX: force all redo at the end // -- restart + seqscan won't read deleted stuff @@ -58,7 +58,7 @@ fn test_regress() { let node = compute_cplane.new_node(); node.start(); - control_plane::regress_check(&node); + control_plane::storage::regress_check(&node); } // Run two postgres instances on one pageserver diff --git a/integration_tests/tests/test_wal_acceptor.rs b/integration_tests/tests/test_wal_acceptor.rs index fbf474f27f..c0fe6c44b9 100644 --- a/integration_tests/tests/test_wal_acceptor.rs +++ b/integration_tests/tests/test_wal_acceptor.rs @@ -1,6 +1,6 @@ // Restart acceptors one by one while compute is under the load. -use control_plane::ComputeControlPlane; -use control_plane::TestStorageControlPlane; +use control_plane::compute::ComputeControlPlane; +use control_plane::storage::TestStorageControlPlane; use rand::Rng; use std::sync::Arc; From 39ebec51d13717c3613ad8afbd528df288217e68 Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Sat, 10 Apr 2021 19:03:40 +0300 Subject: [PATCH 03/33] Return Result<()> from pageserver start/stop. To provide meaningful error messages when it is called by CLI. --- control_plane/src/local_env.rs | 15 ------- control_plane/src/storage.rs | 81 ++++++++++++++++++++++------------ zenith/src/main.rs | 43 ++++-------------- 3 files changed, 61 insertions(+), 78 deletions(-) diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 27f79a644c..aea884026a 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -7,7 +7,6 @@ use std::env; use std::error; use std::fs; -use std::net::SocketAddr; use std::path::{Path, PathBuf}; use home; @@ -34,9 +33,6 @@ pub struct LocalEnv { // Path to pageserver binary. pub zenith_distrib_dir: PathBuf, - - // PageServer-specific options. - pub pageserver: PageServerConf, } impl LocalEnv { @@ -58,11 +54,6 @@ impl LocalEnv { } } -#[derive(Serialize, Deserialize, Clone)] -pub struct PageServerConf { - pub listen_address: SocketAddr, -} - // // Issues in rust-lang repo has several discussions about proper library to check // home directory in a cross-platform way. Seems that current consensus is around @@ -150,9 +141,6 @@ pub fn init() -> Result<()> { data_dir, pg_distrib_dir, zenith_distrib_dir, - pageserver: PageServerConf { - listen_address: "127.0.0.1:5430".parse().unwrap(), - }, }; let toml = toml::to_string(&conf)?; fs::write(cfg_path, toml)?; @@ -191,9 +179,6 @@ pub fn test_env() -> LocalEnv { data_dir, pg_distrib_dir: Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_install"), zenith_distrib_dir: cargo_bin_dir(), - pageserver: PageServerConf { - listen_address: "127.0.0.1:65200".parse().unwrap(), - }, } } diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 76c6257ae1..658016329a 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -7,12 +7,15 @@ use std::thread; use std::time::Duration; use std::path::{Path, PathBuf}; use std::net::SocketAddr; +use std::error; use postgres::{Client, NoTls}; use crate::local_env::{self, LocalEnv}; use crate::compute::{PostgresNode}; +type Result = std::result::Result>; + // // Collection of several example deployments useful for tests. // @@ -33,9 +36,10 @@ impl TestStorageControlPlane { let pserver = Arc::new(PageServerNode { env: env.clone(), kill_on_exit: true, + listen_address: None, }); pserver.init(); - pserver.start(); + pserver.start().unwrap(); TestStorageControlPlane { wal_acceptors: Vec::new(), @@ -52,11 +56,12 @@ impl TestStorageControlPlane { pageserver: Arc::new(PageServerNode { env: env.clone(), kill_on_exit: true, + listen_address: None, }), test_done: AtomicBool::new(false), }; cplane.pageserver.init(); - cplane.pageserver.start(); + cplane.pageserver.start().unwrap(); const WAL_ACCEPTOR_PORT: usize = 54321; @@ -105,77 +110,95 @@ impl Drop for TestStorageControlPlane { // pub struct PageServerNode { kill_on_exit: bool, - env: LocalEnv, + listen_address: Option, + pub env: LocalEnv, } impl PageServerNode { + pub fn from_env(env: &LocalEnv) -> PageServerNode { + PageServerNode { + kill_on_exit: false, + listen_address: None, // default + env: env.clone(), + } + } + + pub fn address(&self) -> SocketAddr { + match self.listen_address { + Some(addr) => addr, + None => "127.0.0.1:64000".parse().unwrap() + } + } + pub fn init(&self) { fs::create_dir_all(self.env.pageserver_data_dir()).unwrap(); } - pub fn start(&self) { - println!( - "Starting pageserver at '{}'", - self.env.pageserver.listen_address - ); + pub fn start(&self) -> Result<()> { + println!("Starting pageserver at '{}'", self.address()); let status = Command::new(self.env.zenith_distrib_dir.join("pageserver")) // XXX -> method .args(&["-D", self.env.pageserver_data_dir().to_str().unwrap()]) .args(&[ "-l", - self.env.pageserver.listen_address.to_string().as_str(), + self.address().to_string().as_str(), ]) .arg("-d") .arg("--skip-recovery") .env_clear() .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to start pageserver"); + .status()?; if !status.success() { - panic!("pageserver start failed"); + return Err(Box::::from( + format!("Pageserver failed to start. See '{}' for details.", + self.env.pageserver_log().to_str().unwrap()) + )); + } else { + return Ok(()); } } - pub fn stop(&self) { + pub fn stop(&self) -> Result<()> { let pidfile = self.env.pageserver_pidfile(); let pid = fs::read_to_string(pidfile).unwrap(); let status = Command::new("kill") - .arg(pid) + .arg(pid.clone()) .env_clear() .status() .expect("failed to execute kill"); if !status.success() { - panic!("kill start failed"); + return Err(Box::::from( + format!("Failed to kill pageserver with pid {}", + pid + ))); } // await for pageserver stop for _ in 0..5 { - let stream = TcpStream::connect(self.env.pageserver.listen_address); + let stream = TcpStream::connect(self.address()); if let Err(_e) = stream { - return; + return Ok(()); } - println!( - "Stopping pageserver on {}", - self.env.pageserver.listen_address - ); + println!("Stopping pageserver on {}", self.address()); thread::sleep(Duration::from_secs(1)); } // ok, we failed to stop pageserver, let's panic - panic!("Failed to stop pageserver"); - } - - pub fn address(&self) -> &std::net::SocketAddr { - &self.env.pageserver.listen_address + if !status.success() { + return Err(Box::::from( + format!("Failed to stop pageserver with pid {}", + pid + ))); + } else { + return Ok(()); + } } pub fn page_server_psql(&self, sql: &str) -> Vec { - // let addr = &self.page_servers[0].env.pageserver.listen_address; - let connstring = format!( "host={} port={} dbname={} user={}", self.address().ip(), @@ -193,7 +216,7 @@ impl PageServerNode { impl Drop for PageServerNode { fn drop(&mut self) { if self.kill_on_exit { - self.stop(); + let _ = self.stop(); } } } diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 20b48ba5f1..d646a1ac29 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -1,9 +1,7 @@ use clap::{App, SubCommand}; -use std::fs; use std::process::exit; -use std::process::Command; -use control_plane::local_env; +use control_plane::{local_env, storage}; fn main() { let matches = App::new("zenith") @@ -44,7 +42,7 @@ fn main() { } // all other commands would need config - let conf = match local_env::load_config() { + let env = match local_env::load_config() { Ok(conf) => conf, Err(e) => { eprintln!("Error loading config from ~/.zenith: {}", e); @@ -58,45 +56,22 @@ fn main() { } ("start", Some(_sub_m)) => { - println!( - "Starting pageserver at '{}'", - conf.pageserver.listen_address - ); + let pageserver = storage::PageServerNode::from_env(&env); - let status = Command::new(conf.zenith_distrib_dir.join("pageserver")) - .args(&["-D", conf.data_dir.to_str().unwrap()]) - .args(&["-l", conf.pageserver.listen_address.to_string().as_str()]) - .arg("-d") - .arg("--skip-recovery") - .env_clear() - .env("PATH", conf.pg_bin_dir().to_str().unwrap()) // pageserver needs postres-wal-redo binary - .env("LD_LIBRARY_PATH", conf.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to start pageserver"); - - if !status.success() { - eprintln!( - "Pageserver failed to start. See '{}' for details.", - conf.pageserver_log().to_str().unwrap() - ); + if let Err(e) = pageserver.start() { + eprintln!("start: {}", e); exit(1); } - // TODO: check it's actually started, or run status - + // TODO: check and await actual start println!("Done!"); } ("stop", Some(_sub_m)) => { - let pid = fs::read_to_string(conf.pageserver_pidfile()).unwrap(); - let status = Command::new("kill") - .arg(pid) - .env_clear() - .status() - .expect("failed to execute kill"); + let pageserver = storage::PageServerNode::from_env(&env); - if !status.success() { - eprintln!("Failed to kill pageserver"); + if let Err(e) = pageserver.stop() { + eprintln!("stop: {}", e); exit(1); } From c5f379bff378937f617af3f386585ea32cb6f339 Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Tue, 13 Apr 2021 09:47:16 +0300 Subject: [PATCH 04/33] [WIP] Implement CLI pg part --- .github/workflows/testing.yml | 1 + Cargo.lock | 2 + control_plane/Cargo.toml | 2 + control_plane/src/compute.rs | 389 ++++++++++--------- control_plane/src/local_env.rs | 57 ++- integration_tests/tests/test_pageserver.rs | 16 +- integration_tests/tests/test_wal_acceptor.rs | 16 +- zenith/src/main.rs | 85 +++- 8 files changed, 327 insertions(+), 241 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 1efc44dedf..2f8acc4385 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -4,6 +4,7 @@ on: [push] jobs: regression-check: + timeout-minutes: 10 name: run regression test suite runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 04d3842934..921924bed7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,9 +383,11 @@ name = "control_plane" version = "0.1.0" dependencies = [ "home", + "lazy_static", "pageserver", "postgres", "rand 0.8.3", + "regex", "serde", "serde_derive", "tokio-postgres", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index dac78ea356..4699a4da40 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -15,6 +15,8 @@ serde = "" serde_derive = "" toml = "" home = "0.5.3" +lazy_static = "" +regex = "1" pageserver = { path = "../pageserver" } walkeeper = { path = "../walkeeper" } diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index bb7a177893..9de4a3b8c6 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -1,199 +1,133 @@ -use std::path::PathBuf; +use std::{collections::BTreeMap, path::PathBuf}; use std::sync::Arc; use std::fs::{self, OpenOptions}; -use std::process::Command; +use std::process::{Command, Stdio}; use std::fs::File; -use std::{ - io::Write, - net::{IpAddr, Ipv4Addr, SocketAddr}, -}; +use std::time::Duration; +use std::{io::Write, net::SocketAddr}; +use std::error; +use std::net::TcpStream; use postgres::{Client, NoTls}; +use lazy_static::lazy_static; +use regex::Regex; use crate::local_env::{self, LocalEnv}; use crate::storage::{PageServerNode, WalProposerNode}; +type Result = std::result::Result>; + // // ComputeControlPlane // pub struct ComputeControlPlane { - pg_bin_dir: PathBuf, - work_dir: PathBuf, - last_assigned_port: u16, + base_port: u16, pageserver: Arc, - nodes: Vec>, + pub nodes: BTreeMap>, env: LocalEnv, } impl ComputeControlPlane { + + // Load current nodes with ports from data directories on disk + pub fn load(env: LocalEnv) -> Result { + // TODO: since pageserver do not have config file yet we believe here that + // it is running on default port. Change that when pageserver will have config. + let pageserver = Arc::new(PageServerNode::from_env(&env)); + + let nodes: Result> = fs::read_dir(env.compute_dir()) + .map_err(|e| format!("failed to list {}: {}", env.compute_dir().to_str().unwrap(), e))? + .into_iter() + .map(|f| { + PostgresNode::from_dir_entry(f?, &env, &pageserver) + .map(|node| (node.name.clone(), Arc::new(node)) ) + }) + .collect(); + let nodes = nodes?; + + Ok(ComputeControlPlane { + base_port: 55431, + pageserver, + nodes, + env + }) + } + + fn get_port(&mut self) -> u16 { + 1 + self.nodes + .iter() + .map(|(_name, node)| node.address.port()) + .max() + .unwrap_or(self.base_port) + } + pub fn local(pageserver: &Arc) -> ComputeControlPlane { let env = local_env::test_env(); ComputeControlPlane { - pg_bin_dir: env.pg_bin_dir(), - work_dir: env.data_dir.clone(), - last_assigned_port: 65431, + base_port: 65431, pageserver: Arc::clone(pageserver), - nodes: Vec::new(), + nodes: BTreeMap::new(), env: env.clone(), } } - // TODO: check port availability and - fn get_port(&mut self) -> u16 { - let port = self.last_assigned_port + 1; - self.last_assigned_port += 1; - port - } - - pub fn new_vanilla_node<'a>(&mut self) -> &Arc { + fn new_vanilla_node(&mut self, is_test: bool) -> Result> { // allocate new node entry with generated port - let node_id = self.nodes.len() + 1; - let node = PostgresNode { - _node_id: node_id, - address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), - pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), + let node_id = self.nodes.len() as u32 + 1; + let node = Arc::new(PostgresNode { + name: format!("pg{}", node_id), + address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()), env: self.env.clone(), pageserver: Arc::clone(&self.pageserver), - }; - self.nodes.push(Arc::new(node)); - let node = self.nodes.last().unwrap(); + is_test + }); + node.init_vanilla()?; + self.nodes.insert(node.name.clone(), Arc::clone(&node)); - println!( - "Creating new postgres: path={} port={}", - node.pgdata.to_str().unwrap(), - node.address.port() - ); - - // initialize data directory - fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok(); - let initdb_path = self.pg_bin_dir.join("initdb"); - let initdb = Command::new(initdb_path) - .args(&["-D", node.pgdata.to_str().unwrap()]) - .arg("-N") - .arg("--no-instructions") - .env_clear() - .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to execute initdb"); - - if !initdb.success() { - panic!("initdb failed"); - } - - // // allow local replication connections - // node.append_conf("pg_hba.conf", format!("\ - // host replication all {}/32 sspi include_realm=1 map=regress\n\ - // ", node.ip).as_str()); - - // listen for selected port - node.append_conf( - "postgresql.conf", - format!( - "\ - max_wal_senders = 10\n\ - max_replication_slots = 10\n\ - hot_standby = on\n\ - shared_buffers = 1MB\n\ - max_connections = 100\n\ - wal_level = replica\n\ - listen_addresses = '{address}'\n\ - port = {port}\n\ - ", - address = node.address.ip(), - port = node.address.port() - ) - .as_str(), - ); - - node + Ok(node) } - // Init compute node without files, only datadir structure - // use initdb --compute-node flag and GUC 'computenode_mode' - // to distinguish the node - pub fn new_minimal_node(&mut self) -> &PostgresNode { - // allocate new node entry with generated port - let node_id = self.nodes.len() + 1; - let node = PostgresNode { - _node_id: node_id, - address: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), self.get_port()), - pgdata: self.work_dir.join(format!("compute/pg{}", node_id)), - env: self.env.clone(), - pageserver: Arc::clone(&self.pageserver), - }; - self.nodes.push(Arc::new(node)); - let node = self.nodes.last().unwrap(); - - // initialize data directory w/o files - fs::remove_dir_all(node.pgdata.to_str().unwrap()).ok(); - let initdb_path = self.pg_bin_dir.join("initdb"); - println!("initdb_path: {}", initdb_path.to_str().unwrap()); - let initdb = Command::new(initdb_path) - .args(&["-D", node.pgdata.to_str().unwrap()]) - .arg("-N") - .arg("--no-instructions") - .arg("--compute-node") - .env_clear() - .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to execute initdb"); - - if !initdb.success() { - panic!("initdb failed"); - } - - // listen for selected port - node.append_conf( - "postgresql.conf", - format!( - "\ - max_wal_senders = 10\n\ - max_replication_slots = 10\n\ - hot_standby = on\n\ - shared_buffers = 1MB\n\ - max_connections = 100\n\ - wal_level = replica\n\ - listen_addresses = '{address}'\n\ - port = {port}\n\ - computenode_mode = true\n\ - ", - address = node.address.ip(), - port = node.address.port() - ) - .as_str(), - ); - - node - } - - pub fn new_node(&mut self) -> Arc { + pub fn new_test_node(&mut self) -> Arc { let addr = self.pageserver.address().clone(); - let node = self.new_vanilla_node(); + let node = self.new_vanilla_node(true).unwrap(); // Configure that node to take pages from pageserver - node.append_conf( - "postgresql.conf", - format!( - "\ - page_server_connstring = 'host={} port={}'\n\ - ", + node.append_conf("postgresql.conf", + format!("page_server_connstring = 'host={} port={}'\n", addr.ip(), addr.port() ) .as_str(), ); - node.clone() + node } - pub fn new_master_node(&mut self) -> Arc { - let node = self.new_vanilla_node(); + pub fn new_test_master_node(&mut self) -> Arc { + let node = self.new_vanilla_node(true).unwrap(); node.append_conf( "postgresql.conf", "synchronous_standby_names = 'safekeeper_proxy'\n", ); - node.clone() + + node + } + + pub fn new_node(&mut self) -> Result> { + let addr = self.pageserver.address().clone(); + let node = self.new_vanilla_node(false)?; + + // Configure that node to take pages from pageserver + node.append_conf("postgresql.conf", + format!("page_server_connstring = 'host={} port={}'\n", + addr.ip(), + addr.port() + ) + .as_str(), + ); + + Ok(node) } } @@ -201,32 +135,144 @@ impl ComputeControlPlane { pub struct PostgresNode { pub address: SocketAddr, - _node_id: usize, - pgdata: PathBuf, + name: String, pub env: LocalEnv, pageserver: Arc, + is_test: bool, } impl PostgresNode { + fn from_dir_entry(entry: std::fs::DirEntry, env: &LocalEnv, pageserver: &Arc) -> Result { + if !entry.file_type()?.is_dir() { + let err_msg = format!( + "PostgresNode::from_dir_entry failed: '{}' is not a directory", + entry.path().to_str().unwrap() + ); + return Err(err_msg.into()); + } + + lazy_static! { + static ref CONF_PORT_RE: Regex = Regex::new(r"(?m)^\s*port\s*=\s*(\d+)\s*$").unwrap(); + } + + // parse data directory name + let fname = entry.file_name(); + let name = fname.to_str().unwrap().to_string(); + + // find out tcp port in config file + let cfg_path = entry.path().join("postgresql.conf"); + let config = fs::read_to_string(cfg_path.clone()) + .map_err(|e| { + format!("failed to read config file in {}: {}", cfg_path.to_str().unwrap(), e) + })?; + + let err_msg = format!("failed to find port definition in config file {}", cfg_path.to_str().unwrap()); + let port: u16 = CONF_PORT_RE + .captures(config.as_str()) + .ok_or(err_msg.clone() + " 1")? + .iter() + .last() + .ok_or(err_msg.clone() + " 3")? + .ok_or(err_msg.clone() + " 3")? + .as_str() + .parse() + .map_err(|e| format!("{}: {}", err_msg, e))?; + + // ok now + Ok(PostgresNode { + address: SocketAddr::new("127.0.0.1".parse().unwrap(), port), + name, + env: env.clone(), + pageserver: Arc::clone(pageserver), + is_test: false + }) + } + + fn init_vanilla(&self) -> Result<()> { + println!( + "Creating new postgres: path={} port={}", + self.pgdata().to_str().unwrap(), + self.address.port() + ); + + // initialize data directory + + if self.is_test { + fs::remove_dir_all(self.pgdata().to_str().unwrap())?; + } + + let initdb_path = self.env.pg_bin_dir().join("initdb"); + let initdb = Command::new(initdb_path) + .args(&["-D", self.pgdata().to_str().unwrap()]) + .arg("-N") + .arg("-A trust") + .arg("--no-instructions") + .env_clear() + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) + .stdout(Stdio::null()) + .status()?; + + if !initdb.success() { + return Err("initdb failed".into()); + } + + // listen for selected port + self.append_conf( + "postgresql.conf", + format!("max_wal_senders = 10\n\ + max_replication_slots = 10\n\ + hot_standby = on\n\ + shared_buffers = 1MB\n\ + max_connections = 100\n\ + wal_level = replica\n\ + listen_addresses = '{address}'\n\ + port = {port}\n", + address = self.address.ip(), + port = self.address.port() + ) + .as_str(), + ); + + println!("Database initialized"); + Ok(()) + } + + fn pgdata(&self) -> PathBuf { + self.env.compute_dir().join(self.name.clone()) + } + + pub fn status(&self) -> &str { + let timeout = Duration::from_millis(300); + let has_pidfile = self.pgdata().join("postmaster.pid").exists(); + let can_connect = TcpStream::connect_timeout(&self.address, timeout).is_ok(); + + match (has_pidfile, can_connect) { + (true, true) => "running", + (false, false) => "stopped", + (true, false) => "crashed", + (false, true) => "running, no pidfile", + } + } + pub fn append_conf(&self, config: &str, opts: &str) { OpenOptions::new() .append(true) - .open(self.pgdata.join(config).to_str().unwrap()) + .open(self.pgdata().join(config).to_str().unwrap()) .unwrap() .write_all(opts.as_bytes()) .unwrap(); } - fn pg_ctl(&self, args: &[&str], check_ok: bool) { + fn pg_ctl(&self, args: &[&str]) -> Result<()> { let pg_ctl_path = self.env.pg_bin_dir().join("pg_ctl"); let pg_ctl = Command::new(pg_ctl_path) .args( [ &[ "-D", - self.pgdata.to_str().unwrap(), + self.pgdata().to_str().unwrap(), "-l", - self.pgdata.join("log").to_str().unwrap(), + self.pgdata().join("log").to_str().unwrap(), ], args, ] @@ -234,28 +280,29 @@ impl PostgresNode { ) .env_clear() .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) - .status() - .expect("failed to execute pg_ctl"); + .status()?; - if check_ok && !pg_ctl.success() { - panic!("pg_ctl failed"); + if !pg_ctl.success() { + Err("pg_ctl failed".into()) + } else { + Ok(()) } } - pub fn start(&self) { + pub fn start(&self) -> Result<()> { let _res = self .pageserver .page_server_psql(format!("callmemaybe {}", self.connstr()).as_str()); println!("Starting postgres node at '{}'", self.connstr()); - self.pg_ctl(&["start"], true); + self.pg_ctl(&["start"]) } - pub fn restart(&self) { - self.pg_ctl(&["restart"], true); + pub fn restart(&self) -> Result<()> { + self.pg_ctl(&["restart"]) } - pub fn stop(&self) { - self.pg_ctl(&["-m", "immediate", "stop"], true); + pub fn stop(&self) -> Result<()> { + self.pg_ctl(&["-m", "immediate", "stop"]) } pub fn connstr(&self) -> String { @@ -305,13 +352,9 @@ impl PostgresNode { Client::connect(connstring.as_str(), NoTls).unwrap() } - pub fn get_pgdata(&self) -> Option<&str> { - self.pgdata.to_str() - } - /* Create stub controlfile and respective xlog to start computenode */ pub fn setup_controlfile(&self) { - let filepath = format!("{}/global/pg_control", self.pgdata.to_str().unwrap()); + let filepath = format!("{}/global/pg_control", self.pgdata().to_str().unwrap()); { File::create(filepath).unwrap(); @@ -320,7 +363,7 @@ impl PostgresNode { let pg_resetwal_path = self.env.pg_bin_dir().join("pg_resetwal"); let pg_resetwal = Command::new(pg_resetwal_path) - .args(&["-D", self.pgdata.to_str().unwrap()]) + .args(&["-D", self.pgdata().to_str().unwrap()]) .arg("-f") // TODO probably we will have to modify pg_resetwal // .arg("--compute-node") @@ -349,7 +392,6 @@ impl PostgresNode { // TODO pub fn pg_bench() {} - pub fn pg_regress() {} } impl Drop for PostgresNode { @@ -357,7 +399,8 @@ impl Drop for PostgresNode { // XXX: we may detect failed test by setting some flag in catch_unwind() // and checking it here. But let just clean datadirs on start. fn drop(&mut self) { - self.stop(); - // fs::remove_dir_all(self.pgdata.clone()).unwrap(); + if self.is_test { + let _ = self.stop(); + } } } diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index aea884026a..4ec4a82a50 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -36,6 +36,7 @@ pub struct LocalEnv { } impl LocalEnv { + // postgres installation pub fn pg_bin_dir(&self) -> PathBuf { self.pg_distrib_dir.join("bin") } @@ -43,6 +44,7 @@ impl LocalEnv { self.pg_distrib_dir.join("lib") } + // pageserver pub fn pageserver_data_dir(&self) -> PathBuf { self.data_dir.join("pageserver") } @@ -52,6 +54,11 @@ impl LocalEnv { pub fn pageserver_pidfile(&self) -> PathBuf { self.pageserver_data_dir().join("pageserver.pid") } + + // compute nodes + pub fn compute_dir(&self) -> PathBuf { + self.data_dir.join("compute") + } } // @@ -60,14 +67,7 @@ impl LocalEnv { // home crate and cargo uses it. // fn get_home() -> Result { - match home::home_dir() { - Some(path) => Ok(path), - None => { - return Err(Box::::from( - "can not determine home directory path", - )); - } - } + home::home_dir().ok_or("can not determine home directory path".into()) } pub fn init() -> Result<()> { @@ -80,7 +80,7 @@ pub fn init() -> Result<()> { "{} already exists. Perhaps already initialized?", cfg_path.to_str().unwrap() ); - return Err(Box::::from(err_msg)); + return Err(err_msg.into()); } // Now we can run init only from crate directory, so check that current dir is our crate. @@ -89,7 +89,7 @@ pub fn init() -> Result<()> { if !cargo_path.join("pageserver/Cargo.toml").exists() { let err_msg = "Current dirrectory does not look like a zenith repo. \ Please, run 'init' from zenith repo root."; - return Err(Box::::from(err_msg)); + return Err(err_msg.into()); } // ok, now check that expected binaries are present @@ -103,7 +103,7 @@ pub fn init() -> Result<()> { Perhaps './pgbuild.sh' is needed to build it first.", pg_path.to_str().unwrap() ); - return Err(Box::::from(err_msg)); + return Err(err_msg.into()); } // check pageserver @@ -114,26 +114,23 @@ pub fn init() -> Result<()> { "Can't find pageserver binary at {}. Please build it.", pageserver_path.to_str().unwrap() ); - return Err(Box::::from(err_msg)); + return Err(err_msg.into()); } // ok, we are good to go - // create data dir - let data_dir = cargo_path.join("tmp_install"); - match fs::create_dir(data_dir.clone()) { - Ok(_) => {} - Err(e) => match e.kind() { - std::io::ErrorKind::AlreadyExists => {} - _ => { - let err_msg = format!( - "Failed to create data directory in '{}': {}", + // create dirs + let data_dir = cargo_path.join("tmp_check"); + + for &dir in &["compute", "pageserver"] { + fs::create_dir_all(data_dir.join(dir)) + .map_err(|e| { + format!( + "Failed to create directory in '{}': {}", data_dir.to_str().unwrap(), e - ); - return Err(Box::::from(err_msg)); - } - }, + ) + })?; } // write config @@ -160,15 +157,13 @@ pub fn load_config() -> Result { "Zenith config is not found in {}. You need to run 'zenith init' first", cfg_path.to_str().unwrap() ); - return Err(Box::::from(err_msg)); + return Err(err_msg.into()); } // load and parse file let config = fs::read_to_string(cfg_path)?; - match toml::from_str(config.as_str()) { - Ok(cfg) => Ok(cfg), - Err(e) => Err(Box::::from(e)), - } + toml::from_str(config.as_str()) + .map_err(|e| e.into()) } // local env for tests @@ -184,7 +179,7 @@ pub fn test_env() -> LocalEnv { // Find the directory where the binaries were put (i.e. target/debug/) pub fn cargo_bin_dir() -> PathBuf { - let mut pathbuf = std::env::current_exe().ok().unwrap(); + let mut pathbuf = std::env::current_exe().unwrap(); pathbuf.pop(); if pathbuf.ends_with("deps") { diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index 2322f4d61e..51f9ba9ddb 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -15,8 +15,8 @@ fn test_redo_cases() { let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // start postgres - let node = compute_cplane.new_node(); - node.start(); + let node = compute_cplane.new_test_node(); + node.start().unwrap(); // check basic work with table node.safe_psql( @@ -55,8 +55,8 @@ fn test_regress() { let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // start postgres - let node = compute_cplane.new_node(); - node.start(); + let node = compute_cplane.new_test_node(); + node.start().unwrap(); control_plane::storage::regress_check(&node); } @@ -69,10 +69,10 @@ fn test_pageserver_multitenancy() { let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // Allocate postgres instance, but don't start - let node1 = compute_cplane.new_node(); - let node2 = compute_cplane.new_node(); - node1.start(); - node2.start(); + let node1 = compute_cplane.new_test_node(); + let node2 = compute_cplane.new_test_node(); + node1.start().unwrap(); + node2.start().unwrap(); // check node1 node1.safe_psql( diff --git a/integration_tests/tests/test_wal_acceptor.rs b/integration_tests/tests/test_wal_acceptor.rs index c0fe6c44b9..b826bd32ea 100644 --- a/integration_tests/tests/test_wal_acceptor.rs +++ b/integration_tests/tests/test_wal_acceptor.rs @@ -16,8 +16,8 @@ fn test_acceptors_normal_work() { let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); // start postgres - let node = compute_cplane.new_master_node(); - node.start(); + let node = compute_cplane.new_test_master_node(); + node.start().unwrap(); // start proxy let _proxy = node.start_proxy(wal_acceptors); @@ -54,8 +54,8 @@ fn test_acceptors_restarts() { let mut rng = rand::thread_rng(); // start postgres - let node = compute_cplane.new_master_node(); - node.start(); + let node = compute_cplane.new_test_master_node(); + node.start().unwrap(); // start proxy let _proxy = node.start_proxy(wal_acceptors); @@ -112,8 +112,8 @@ fn test_acceptors_unavalability() { let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); // start postgres - let node = compute_cplane.new_master_node(); - node.start(); + let node = compute_cplane.new_test_master_node(); + node.start().unwrap(); // start proxy let _proxy = node.start_proxy(wal_acceptors); @@ -187,8 +187,8 @@ fn test_race_conditions() { let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); // start postgres - let node = compute_cplane.new_master_node(); - node.start(); + let node = compute_cplane.new_test_master_node(); + node.start().unwrap(); // start proxy let _proxy = node.start_proxy(wal_acceptors); diff --git a/zenith/src/main.rs b/zenith/src/main.rs index d646a1ac29..3c3785b748 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -1,9 +1,17 @@ -use clap::{App, SubCommand}; +use clap::{App, ArgMatches, SubCommand, Arg}; use std::process::exit; +use std::error; -use control_plane::{local_env, storage}; +use control_plane::{compute::ComputeControlPlane, local_env, storage}; + +type Result = std::result::Result>; fn main() { + let name_arg = Arg::with_name("NAME") + .short("n") + .index(1) + .help("name of this postgres instance") + .required(true); let matches = App::new("zenith") .subcommand(SubCommand::with_name("init")) .subcommand(SubCommand::with_name("start")) @@ -12,10 +20,18 @@ fn main() { .subcommand( SubCommand::with_name("pg") .about("Manage postgres instances") - .subcommand(SubCommand::with_name("create")) - .subcommand(SubCommand::with_name("start")) - .subcommand(SubCommand::with_name("stop")) - .subcommand(SubCommand::with_name("destroy")), + .subcommand(SubCommand::with_name("create") + // .arg(name_arg.clone() + // .required(false) + // .help("name of this postgres instance (will be pgN if omitted)")) + ) + .subcommand(SubCommand::with_name("list")) + .subcommand(SubCommand::with_name("start") + .arg(name_arg.clone())) + .subcommand(SubCommand::with_name("stop") + .arg(name_arg.clone())) + .subcommand(SubCommand::with_name("destroy") + .arg(name_arg.clone())) ) .subcommand( SubCommand::with_name("snapshot") @@ -52,43 +68,70 @@ fn main() { match matches.subcommand() { ("init", Some(_)) => { - panic!() /* init was handled before */ + panic!() /* Should not happen. Init was handled before */ } ("start", Some(_sub_m)) => { let pageserver = storage::PageServerNode::from_env(&env); if let Err(e) = pageserver.start() { - eprintln!("start: {}", e); + eprintln!("pageserver start: {}", e); exit(1); } - - // TODO: check and await actual start - println!("Done!"); } ("stop", Some(_sub_m)) => { let pageserver = storage::PageServerNode::from_env(&env); - if let Err(e) = pageserver.stop() { - eprintln!("stop: {}", e); + eprintln!("pageserver stop: {}", e); exit(1); } - - println!("Done!"); } ("status", Some(_sub_m)) => {} ("pg", Some(pg_match)) => { - match pg_match.subcommand() { - ("start", Some(_sub_m)) => { - println!("xxx: pg start"); - // Ok(()) - } - _ => {} + if let Err(e) = handle_pg(pg_match, &env){ + eprintln!("pg operation failed: {}", e); + exit(1); } } _ => {} } } + +fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { + let mut cplane = ComputeControlPlane::load(env.clone())?; + + match pg_match.subcommand() { + ("create", Some(_sub_m)) => { + cplane.new_node()?; + } + ("list", Some(_sub_m)) => { + println!("NODE\tADDRESS\tSTATUS"); + for (node_name, node) in cplane.nodes.iter() { + println!("{}\t{}\t{}", node_name, node.address, node.status()); + } + } + ("start", Some(sub_m)) => { + let name = sub_m.value_of("NAME").unwrap(); + let node = cplane + .nodes + .get(name) + .ok_or(format!("postgres {} is not found", name))?; + node.start()?; + } + ("stop", Some(sub_m)) => { + let name = sub_m.value_of("NAME").unwrap(); + let node = cplane + .nodes + .get(name) + .ok_or(format!("postgres {} is not found", name))?; + node.stop()?; + } + + _ => {} + } + + Ok(()) +} From f35d13183e626a5ce37d2d2a2443354d0423a5da Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Tue, 13 Apr 2021 18:36:02 +0300 Subject: [PATCH 05/33] fixup, check testing in CI --- .gitignore | 3 ++- control_plane/src/compute.rs | 4 +++- control_plane/src/local_env.rs | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 20348359a4..7d9bf1115c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target -/tmp_check/ +/tmp_check /tmp_install +/tmp_check_cli diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 9de4a3b8c6..10e75ec5cb 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -198,9 +198,11 @@ impl PostgresNode { // initialize data directory if self.is_test { - fs::remove_dir_all(self.pgdata().to_str().unwrap())?; + fs::remove_dir_all(self.pgdata().to_str().unwrap()).ok(); } + fs::create_dir_all(self.pgdata().to_str().unwrap())?; + let initdb_path = self.env.pg_bin_dir().join("initdb"); let initdb = Command::new(initdb_path) .args(&["-D", self.pgdata().to_str().unwrap()]) diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 4ec4a82a50..265d28629e 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -120,7 +120,7 @@ pub fn init() -> Result<()> { // ok, we are good to go // create dirs - let data_dir = cargo_path.join("tmp_check"); + let data_dir = cargo_path.join("tmp_check_cli"); for &dir in &["compute", "pageserver"] { fs::create_dir_all(data_dir.join(dir)) @@ -168,7 +168,7 @@ pub fn load_config() -> Result { // local env for tests pub fn test_env() -> LocalEnv { - let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tmp_check"); + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../tmp_check"); fs::create_dir_all(data_dir.clone()).unwrap(); LocalEnv { data_dir, From b07fa4c8963fd599482f4de201c8291c1f76c304 Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Tue, 13 Apr 2021 18:57:46 +0300 Subject: [PATCH 06/33] update readme --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/README.md b/README.md index a9e6633f45..b7c745bcb8 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,54 @@ Zenith substitutes PostgreSQL storage layer and redistributes data across a cluster of nodes +## Running local installation + +1. Build zenith and patched postgres +```sh +git clone --recursive https://github.com/libzenith/zenith.git +cd zenith +./pgbuild.sh # builds postgres and installs it to ./tmp_install +cargo build +``` + +2. Start pageserver and postggres on top of it (should be called from repo root): +```sh +# Create ~/.zenith with proper paths to binaries and data +# Later that would be responsibility of a package install script +>./target/debug/zenith init + +# start pageserver +> ./target/debug/zenith start +Starting pageserver at '127.0.0.1:64000' + +# create and configure postgres data dir +> ./target/debug/zenith pg create +Creating new postgres: path=/Users/user/code/zenith/tmp_check_cli/compute/pg1 port=55432 +Database initialized + +# start it +> ./target/debug/zenith pg start pg1 + +# look up status and connection info +> ./target/debug/zenith pg list +NODE ADDRESS STATUS +pg1 127.0.0.1:55432 running +``` + +3. Now it is possible to connect to postgres and run some queries: +``` +> psql -p55432 -h 127.0.0.1 postgres +postgres=# CREATE TABLE t(key int primary key, value text); +CREATE TABLE +postgres=# insert into t values(1,1); +INSERT 0 1 +postgres=# select * from t; + key | value +-----+------- + 1 | 1 +(1 row) +``` + ## Running tests ```sh From 46543f54a6edfb4c3af9be0d15a472623a8f051c Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Tue, 13 Apr 2021 11:43:54 -0700 Subject: [PATCH 07/33] pgbuild.sh: halt if a subcommand fails This is helpful when first setting up a build machine, just in case build dependencies are missing. --- pgbuild.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pgbuild.sh b/pgbuild.sh index 0514a5868a..9d4c0baa65 100755 --- a/pgbuild.sh +++ b/pgbuild.sh @@ -10,6 +10,10 @@ # # 2) installs postgres to REPO_ROOT/tmp_install/ # + +# Halt immediately if any command fails +set -e + REPO_ROOT=$(dirname "$0") REPO_ROOT="`( cd \"$REPO_ROOT\" && pwd )`" From 3c4ebc4030af26e841a531aeb6fe26fae9d8d732 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Tue, 13 Apr 2021 14:06:14 -0700 Subject: [PATCH 08/33] init_logging: return Result, print error on file create Instead of panicking if the file create fails, print the filename and error description to stderr; then propagate the error to our caller. --- pageserver/src/bin/pageserver.rs | 16 ++++++++++------ walkeeper/src/bin/wal_acceptor.rs | 14 +++++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 9fdf405145..2f34bb7a6f 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -102,7 +102,7 @@ fn main() -> Result<(), io::Error> { fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> { // Initialize logger - let _scope_guard = init_logging(&conf); + let _scope_guard = init_logging(&conf)?; let _log_guard = slog_stdlog::init().unwrap(); // Note: this `info!(...)` macro comes from `log` crate @@ -217,12 +217,16 @@ fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> { Ok(()) } -fn init_logging(conf: &PageServerConf) -> slog_scope::GlobalLoggerGuard { +fn init_logging(conf: &PageServerConf) -> Result { if conf.interactive { - tui::init_logging() + Ok(tui::init_logging()) } else if conf.daemonize { let log = conf.data_dir.join("pageserver.log"); - let log_file = File::create(log).unwrap_or_else(|_| panic!("Could not create log file")); + let log_file = File::create(&log).map_err(|err| { + // We failed to initialize logging, so we can't log this message with error! + eprintln!("Could not create log file {:?}: {}", log, err); + err + })?; let decorator = slog_term::PlainSyncDecorator::new(log_file); let drain = slog_term::CompactFormat::new(decorator).build(); let drain = slog::Filter::new(drain, |record: &slog::Record| { @@ -233,7 +237,7 @@ fn init_logging(conf: &PageServerConf) -> slog_scope::GlobalLoggerGuard { }); let drain = std::sync::Mutex::new(drain).fuse(); let logger = slog::Logger::root(drain, slog::o!()); - slog_scope::set_global_logger(logger) + Ok(slog_scope::set_global_logger(logger)) } else { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse(); @@ -251,6 +255,6 @@ fn init_logging(conf: &PageServerConf) -> slog_scope::GlobalLoggerGuard { }) .fuse(); let logger = slog::Logger::root(drain, slog::o!()); - slog_scope::set_global_logger(logger) + Ok(slog_scope::set_global_logger(logger)) } } diff --git a/walkeeper/src/bin/wal_acceptor.rs b/walkeeper/src/bin/wal_acceptor.rs index 75466bc328..c8d635ba00 100644 --- a/walkeeper/src/bin/wal_acceptor.rs +++ b/walkeeper/src/bin/wal_acceptor.rs @@ -92,7 +92,7 @@ fn main() -> Result<(), io::Error> { fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<(), io::Error> { // Initialize logger - let _scope_guard = init_logging(&conf); + let _scope_guard = init_logging(&conf)?; let _log_guard = slog_stdlog::init().unwrap(); // Note: this `info!(...)` macro comes from `log` crate info!("standard logging redirected to slog"); @@ -141,20 +141,24 @@ fn start_wal_acceptor(conf: WalAcceptorConf) -> Result<(), io::Error> { Ok(()) } -fn init_logging(conf: &WalAcceptorConf) -> slog_scope::GlobalLoggerGuard { +fn init_logging(conf: &WalAcceptorConf) -> Result { if conf.daemonize { let log = conf.data_dir.join("wal_acceptor.log"); - let log_file = File::create(log).unwrap_or_else(|_| panic!("Could not create log file")); + let log_file = File::create(&log).map_err(|err| { + // We failed to initialize logging, so we can't log this message with error! + eprintln!("Could not create log file {:?}: {}", log, err); + err + })?; let decorator = slog_term::PlainSyncDecorator::new(log_file); let drain = slog_term::CompactFormat::new(decorator).build(); let drain = std::sync::Mutex::new(drain).fuse(); let logger = slog::Logger::root(drain, slog::o!()); - slog_scope::set_global_logger(logger) + Ok(slog_scope::set_global_logger(logger)) } else { let decorator = slog_term::TermDecorator::new().build(); let drain = slog_term::FullFormat::new(decorator).build().fuse(); let drain = slog_async::Async::new(drain).chan_size(1000).build().fuse(); let logger = slog::Logger::root(drain, slog::o!()); - return slog_scope::set_global_logger(logger); + Ok(slog_scope::set_global_logger(logger)) } } From d1d6c968d598baa16465a60ecea51eed86ab37e3 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Tue, 13 Apr 2021 14:26:34 -0700 Subject: [PATCH 09/33] control_plane: add error handling to reading pid files print file errors to stderr; propagate the io::Error to the caller. This error isn't handled very gracefully in WalAcceptorNode::drop(), but there aren't any good options there since drop can't fail. --- control_plane/src/storage.rs | 36 ++++++++++++++------ integration_tests/tests/test_wal_acceptor.rs | 8 ++--- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 658016329a..3cbd3945ab 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -1,6 +1,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::fs; +use std::io; use std::process::Command; use std::net::TcpStream; use std::thread; @@ -162,10 +163,10 @@ impl PageServerNode { pub fn stop(&self) -> Result<()> { let pidfile = self.env.pageserver_pidfile(); - let pid = fs::read_to_string(pidfile).unwrap(); + let pid = read_pidfile(&pidfile)?; let status = Command::new("kill") - .arg(pid.clone()) + .arg(&pid) .env_clear() .status() .expect("failed to execute kill"); @@ -260,22 +261,24 @@ impl WalAcceptorNode { } } - pub fn stop(&self) { + pub fn stop(&self) -> std::result::Result<(), io::Error> { println!("Stopping wal acceptor on {}", self.listen); let pidfile = self.data_dir.join("wal_acceptor.pid"); - if let Ok(pid) = fs::read_to_string(pidfile) { - let _status = Command::new("kill") - .arg(pid) - .env_clear() - .status() - .expect("failed to execute kill"); - } + let pid = read_pidfile(&pidfile)?; + // Ignores any failures when running this command + let _status = Command::new("kill") + .arg(pid) + .env_clear() + .status() + .expect("failed to execute kill"); + + Ok(()) } } impl Drop for WalAcceptorNode { fn drop(&mut self) { - self.stop(); + self.stop().unwrap(); } } @@ -340,3 +343,14 @@ pub fn regress_check(pg: &PostgresNode) { .status() .expect("pg_regress failed"); } + +/// Read a PID file +/// +/// This should contain an unsigned integer, but we return it as a String +/// because our callers only want to pass it back into a subcommand. +fn read_pidfile(pidfile: &Path) -> std::result::Result { + fs::read_to_string(pidfile).map_err(|err| { + eprintln!("failed to read pidfile {:?}: {:?}", pidfile, err); + err + }) +} diff --git a/integration_tests/tests/test_wal_acceptor.rs b/integration_tests/tests/test_wal_acceptor.rs index b826bd32ea..523c568bb4 100644 --- a/integration_tests/tests/test_wal_acceptor.rs +++ b/integration_tests/tests/test_wal_acceptor.rs @@ -78,7 +78,7 @@ fn test_acceptors_restarts() { } else { let node: usize = rng.gen_range(0..REDUNDANCY); failed_node = Some(node); - storage_cplane.wal_acceptors[node].stop(); + storage_cplane.wal_acceptors[node].stop().unwrap(); } } } @@ -127,7 +127,7 @@ fn test_acceptors_unavalability() { psql.execute("INSERT INTO t values (1, 'payload')", &[]) .unwrap(); - storage_cplane.wal_acceptors[0].stop(); + storage_cplane.wal_acceptors[0].stop().unwrap(); let cp = Arc::new(storage_cplane); start_acceptor(&cp, 0); let now = SystemTime::now(); @@ -137,7 +137,7 @@ fn test_acceptors_unavalability() { psql.execute("INSERT INTO t values (3, 'payload')", &[]) .unwrap(); - cp.wal_acceptors[1].stop(); + cp.wal_acceptors[1].stop().unwrap(); start_acceptor(&cp, 1); psql.execute("INSERT INTO t values (4, 'payload')", &[]) .unwrap(); @@ -164,7 +164,7 @@ fn simulate_failures(cplane: Arc) { let mask: u32 = rng.gen_range(0..(1 << n_acceptors)); for i in 0..n_acceptors { if (mask & (1 << i)) != 0 { - cplane.wal_acceptors[i].stop(); + cplane.wal_acceptors[i].stop().unwrap(); } } thread::sleep(failure_period); From 6266fd102cde045653538f21378a9297754350f3 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 14 Apr 2021 18:18:38 +0300 Subject: [PATCH 10/33] Avoid short writes if a buffer is full. write_buf() tries to write as much as it can in one go, and can return without writing the whole buffer. We need to use write_all() instead. --- pageserver/src/page_service.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 06760c6f68..fcaf8c11f6 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -317,7 +317,7 @@ impl Connection { .await?; self.stream.write_i16(1).await?; - self.stream.write_buf(&mut b).await?; + self.stream.write_all(&mut b).await?; self.stream.write_i32(0).await?; /* table oid */ self.stream.write_i16(0).await?; /* attnum */ self.stream.write_i32(25).await?; /* TEXTOID */ @@ -336,7 +336,7 @@ impl Connection { self.stream.write_i16(1).await?; self.stream.write_i32(b.len() as i32).await?; - self.stream.write_buf(&mut b).await?; + self.stream.write_all(&mut b).await?; } BeMessage::ControlFile => { @@ -348,7 +348,7 @@ impl Connection { self.stream.write_i16(1).await?; self.stream.write_i32(b.len() as i32).await?; - self.stream.write_buf(&mut b).await?; + self.stream.write_all(&mut b).await?; } BeMessage::CommandComplete => { @@ -356,7 +356,7 @@ impl Connection { self.stream.write_u8(b'C').await?; self.stream.write_i32(4 + b.len() as i32).await?; - self.stream.write_buf(&mut b).await?; + self.stream.write_all(&mut b).await?; } BeMessage::ZenithStatusResponse(resp) => { @@ -383,7 +383,7 @@ impl Connection { self.stream.write_u8(102).await?; /* tag from pagestore_client.h */ self.stream.write_u8(resp.ok as u8).await?; self.stream.write_u32(resp.n_blocks).await?; - self.stream.write_buf(&mut resp.page.clone()).await?; + self.stream.write_all(&mut resp.page.clone()).await?; } } @@ -404,7 +404,7 @@ impl Connection { match m.kind { StartupRequestCode::NegotiateGss | StartupRequestCode::NegotiateSsl => { let mut b = Bytes::from("N"); - self.stream.write_buf(&mut b).await?; + self.stream.write_all(&mut b).await?; self.stream.flush().await?; } StartupRequestCode::Normal => { From 2e9c730dd17dfe2821b304f5a7f7622abd2537e1 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 14 Apr 2021 20:12:36 +0300 Subject: [PATCH 11/33] Cargo fmt pass --- control_plane/src/compute.rs | 73 ++++++++++++++++++++++------------ control_plane/src/lib.rs | 2 +- control_plane/src/local_env.rs | 18 ++++----- control_plane/src/storage.rs | 37 ++++++++--------- pageserver/src/page_cache.rs | 10 +++-- pageserver/src/walredo.rs | 8 ++-- zenith/src/main.rs | 25 +++++------- 7 files changed, 96 insertions(+), 77 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 10e75ec5cb..205a1889e2 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -1,15 +1,15 @@ -use std::{collections::BTreeMap, path::PathBuf}; -use std::sync::Arc; -use std::fs::{self, OpenOptions}; -use std::process::{Command, Stdio}; -use std::fs::File; -use std::time::Duration; -use std::{io::Write, net::SocketAddr}; use std::error; +use std::fs::File; +use std::fs::{self, OpenOptions}; use std::net::TcpStream; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::time::Duration; +use std::{collections::BTreeMap, path::PathBuf}; +use std::{io::Write, net::SocketAddr}; -use postgres::{Client, NoTls}; use lazy_static::lazy_static; +use postgres::{Client, NoTls}; use regex::Regex; use crate::local_env::{self, LocalEnv}; @@ -28,7 +28,6 @@ pub struct ComputeControlPlane { } impl ComputeControlPlane { - // Load current nodes with ports from data directories on disk pub fn load(env: LocalEnv) -> Result { // TODO: since pageserver do not have config file yet we believe here that @@ -36,11 +35,17 @@ impl ComputeControlPlane { let pageserver = Arc::new(PageServerNode::from_env(&env)); let nodes: Result> = fs::read_dir(env.compute_dir()) - .map_err(|e| format!("failed to list {}: {}", env.compute_dir().to_str().unwrap(), e))? + .map_err(|e| { + format!( + "failed to list {}: {}", + env.compute_dir().to_str().unwrap(), + e + ) + })? .into_iter() .map(|f| { PostgresNode::from_dir_entry(f?, &env, &pageserver) - .map(|node| (node.name.clone(), Arc::new(node)) ) + .map(|node| (node.name.clone(), Arc::new(node))) }) .collect(); let nodes = nodes?; @@ -49,12 +54,13 @@ impl ComputeControlPlane { base_port: 55431, pageserver, nodes, - env + env, }) } fn get_port(&mut self) -> u16 { - 1 + self.nodes + 1 + self + .nodes .iter() .map(|(_name, node)| node.address.port()) .max() @@ -79,7 +85,7 @@ impl ComputeControlPlane { address: SocketAddr::new("127.0.0.1".parse().unwrap(), self.get_port()), env: self.env.clone(), pageserver: Arc::clone(&self.pageserver), - is_test + is_test, }); node.init_vanilla()?; self.nodes.insert(node.name.clone(), Arc::clone(&node)); @@ -92,8 +98,10 @@ impl ComputeControlPlane { let node = self.new_vanilla_node(true).unwrap(); // Configure that node to take pages from pageserver - node.append_conf("postgresql.conf", - format!("page_server_connstring = 'host={} port={}'\n", + node.append_conf( + "postgresql.conf", + format!( + "page_server_connstring = 'host={} port={}'\n", addr.ip(), addr.port() ) @@ -119,8 +127,10 @@ impl ComputeControlPlane { let node = self.new_vanilla_node(false)?; // Configure that node to take pages from pageserver - node.append_conf("postgresql.conf", - format!("page_server_connstring = 'host={} port={}'\n", + node.append_conf( + "postgresql.conf", + format!( + "page_server_connstring = 'host={} port={}'\n", addr.ip(), addr.port() ) @@ -142,7 +152,11 @@ pub struct PostgresNode { } impl PostgresNode { - fn from_dir_entry(entry: std::fs::DirEntry, env: &LocalEnv, pageserver: &Arc) -> Result { + fn from_dir_entry( + entry: std::fs::DirEntry, + env: &LocalEnv, + pageserver: &Arc, + ) -> Result { if !entry.file_type()?.is_dir() { let err_msg = format!( "PostgresNode::from_dir_entry failed: '{}' is not a directory", @@ -161,12 +175,18 @@ impl PostgresNode { // find out tcp port in config file let cfg_path = entry.path().join("postgresql.conf"); - let config = fs::read_to_string(cfg_path.clone()) - .map_err(|e| { - format!("failed to read config file in {}: {}", cfg_path.to_str().unwrap(), e) - })?; + let config = fs::read_to_string(cfg_path.clone()).map_err(|e| { + format!( + "failed to read config file in {}: {}", + cfg_path.to_str().unwrap(), + e + ) + })?; - let err_msg = format!("failed to find port definition in config file {}", cfg_path.to_str().unwrap()); + let err_msg = format!( + "failed to find port definition in config file {}", + cfg_path.to_str().unwrap() + ); let port: u16 = CONF_PORT_RE .captures(config.as_str()) .ok_or(err_msg.clone() + " 1")? @@ -184,7 +204,7 @@ impl PostgresNode { name, env: env.clone(), pageserver: Arc::clone(pageserver), - is_test: false + is_test: false, }) } @@ -221,7 +241,8 @@ impl PostgresNode { // listen for selected port self.append_conf( "postgresql.conf", - format!("max_wal_senders = 10\n\ + format!( + "max_wal_senders = 10\n\ max_replication_slots = 10\n\ hot_standby = on\n\ shared_buffers = 1MB\n\ diff --git a/control_plane/src/lib.rs b/control_plane/src/lib.rs index 2c2857cb64..a49d39150a 100644 --- a/control_plane/src/lib.rs +++ b/control_plane/src/lib.rs @@ -7,6 +7,6 @@ // local installations. // -pub mod local_env; pub mod compute; +pub mod local_env; pub mod storage; diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 265d28629e..03cf982aa8 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -123,14 +123,13 @@ pub fn init() -> Result<()> { let data_dir = cargo_path.join("tmp_check_cli"); for &dir in &["compute", "pageserver"] { - fs::create_dir_all(data_dir.join(dir)) - .map_err(|e| { - format!( - "Failed to create directory in '{}': {}", - data_dir.to_str().unwrap(), - e - ) - })?; + fs::create_dir_all(data_dir.join(dir)).map_err(|e| { + format!( + "Failed to create directory in '{}': {}", + data_dir.to_str().unwrap(), + e + ) + })?; } // write config @@ -162,8 +161,7 @@ pub fn load_config() -> Result { // load and parse file let config = fs::read_to_string(cfg_path)?; - toml::from_str(config.as_str()) - .map_err(|e| e.into()) + toml::from_str(config.as_str()).map_err(|e| e.into()) } // local env for tests diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 3cbd3945ab..4dc214840c 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -1,19 +1,19 @@ -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; +use std::error; use std::fs; use std::io; -use std::process::Command; +use std::net::SocketAddr; use std::net::TcpStream; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use std::thread; use std::time::Duration; -use std::path::{Path, PathBuf}; -use std::net::SocketAddr; -use std::error; use postgres::{Client, NoTls}; +use crate::compute::PostgresNode; use crate::local_env::{self, LocalEnv}; -use crate::compute::{PostgresNode}; type Result = std::result::Result>; @@ -127,7 +127,7 @@ impl PageServerNode { pub fn address(&self) -> SocketAddr { match self.listen_address { Some(addr) => addr, - None => "127.0.0.1:64000".parse().unwrap() + None => "127.0.0.1:64000".parse().unwrap(), } } @@ -140,10 +140,7 @@ impl PageServerNode { let status = Command::new(self.env.zenith_distrib_dir.join("pageserver")) // XXX -> method .args(&["-D", self.env.pageserver_data_dir().to_str().unwrap()]) - .args(&[ - "-l", - self.address().to_string().as_str(), - ]) + .args(&["-l", self.address().to_string().as_str()]) .arg("-d") .arg("--skip-recovery") .env_clear() @@ -152,10 +149,10 @@ impl PageServerNode { .status()?; if !status.success() { - return Err(Box::::from( - format!("Pageserver failed to start. See '{}' for details.", - self.env.pageserver_log().to_str().unwrap()) - )); + return Err(Box::::from(format!( + "Pageserver failed to start. See '{}' for details.", + self.env.pageserver_log().to_str().unwrap() + ))); } else { return Ok(()); } @@ -172,8 +169,8 @@ impl PageServerNode { .expect("failed to execute kill"); if !status.success() { - return Err(Box::::from( - format!("Failed to kill pageserver with pid {}", + return Err(Box::::from(format!( + "Failed to kill pageserver with pid {}", pid ))); } @@ -190,8 +187,8 @@ impl PageServerNode { // ok, we failed to stop pageserver, let's panic if !status.success() { - return Err(Box::::from( - format!("Failed to stop pageserver with pid {}", + return Err(Box::::from(format!( + "Failed to stop pageserver with pid {}", pid ))); } else { diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index 8df35ea534..c514292340 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -277,7 +277,8 @@ impl PageCache { if wait_result.1.timed_out() { return Err(format!( "Timed out while waiting for WAL record at LSN {:X}/{:X} to arrive", - lsn >> 32, lsn & 0xffff_ffff + lsn >> 32, + lsn & 0xffff_ffff ))?; } } @@ -286,8 +287,11 @@ impl PageCache { } if lsn < shared.first_valid_lsn { - return Err(format!("LSN {:X}/{:X} has already been removed", - lsn >> 32, lsn & 0xffff_ffff))?; + return Err(format!( + "LSN {:X}/{:X} has already been removed", + lsn >> 32, + lsn & 0xffff_ffff + ))?; } let pagecache = &shared.pagecache; diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 2eb1ea3a57..685e771f4a 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -160,9 +160,11 @@ impl WalRedoProcess { .expect("failed to execute initdb"); if !initdb.status.success() { - panic!("initdb failed: {}\nstderr:\n{}", - std::str::from_utf8(&initdb.stdout).unwrap(), - std::str::from_utf8(&initdb.stderr).unwrap()); + panic!( + "initdb failed: {}\nstderr:\n{}", + std::str::from_utf8(&initdb.stdout).unwrap(), + std::str::from_utf8(&initdb.stderr).unwrap() + ); } // Start postgres itself diff --git a/zenith/src/main.rs b/zenith/src/main.rs index 3c3785b748..f6690dd8d7 100644 --- a/zenith/src/main.rs +++ b/zenith/src/main.rs @@ -1,6 +1,6 @@ -use clap::{App, ArgMatches, SubCommand, Arg}; -use std::process::exit; +use clap::{App, Arg, ArgMatches, SubCommand}; use std::error; +use std::process::exit; use control_plane::{compute::ComputeControlPlane, local_env, storage}; @@ -20,18 +20,15 @@ fn main() { .subcommand( SubCommand::with_name("pg") .about("Manage postgres instances") - .subcommand(SubCommand::with_name("create") - // .arg(name_arg.clone() - // .required(false) - // .help("name of this postgres instance (will be pgN if omitted)")) - ) + .subcommand( + SubCommand::with_name("create"), // .arg(name_arg.clone() + // .required(false) + // .help("name of this postgres instance (will be pgN if omitted)")) + ) .subcommand(SubCommand::with_name("list")) - .subcommand(SubCommand::with_name("start") - .arg(name_arg.clone())) - .subcommand(SubCommand::with_name("stop") - .arg(name_arg.clone())) - .subcommand(SubCommand::with_name("destroy") - .arg(name_arg.clone())) + .subcommand(SubCommand::with_name("start").arg(name_arg.clone())) + .subcommand(SubCommand::with_name("stop").arg(name_arg.clone())) + .subcommand(SubCommand::with_name("destroy").arg(name_arg.clone())), ) .subcommand( SubCommand::with_name("snapshot") @@ -91,7 +88,7 @@ fn main() { ("status", Some(_sub_m)) => {} ("pg", Some(pg_match)) => { - if let Err(e) = handle_pg(pg_match, &env){ + if let Err(e) = handle_pg(pg_match, &env) { eprintln!("pg operation failed: {}", e); exit(1); } From 82dc1e82ba550930853e19435cd1e6f138b42168 Mon Sep 17 00:00:00 2001 From: lubennikovaav Date: Wed, 14 Apr 2021 21:14:10 +0300 Subject: [PATCH 12/33] Restore pageserver from s3 or local datadir (#9) * change pageserver --skip-recovery option to --restore-from=[s3|local] * implement restore from local pgdata * add simple test for local restore --- Cargo.lock | 30 +++ control_plane/src/storage.rs | 37 ++- integration_tests/tests/test_pageserver.rs | 7 +- pageserver/Cargo.toml | 1 + pageserver/src/bin/pageserver.rs | 19 +- pageserver/src/lib.rs | 3 +- pageserver/src/restore_datadir.rs | 270 +++++++++++++++++++++ 7 files changed, 352 insertions(+), 15 deletions(-) create mode 100644 pageserver/src/restore_datadir.rs diff --git a/Cargo.lock b/Cargo.lock index 921924bed7..fe4be36d72 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1222,6 +1222,7 @@ dependencies = [ "tokio-postgres", "tokio-stream", "tui", + "walkdir", ] [[package]] @@ -1667,6 +1668,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.19" @@ -2268,6 +2278,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "walkeeper" version = "0.1.0" @@ -2433,6 +2454,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 4dc214840c..86c0d50327 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -31,7 +31,7 @@ pub struct TestStorageControlPlane { impl TestStorageControlPlane { // postgres <-> page_server - pub fn one_page_server() -> TestStorageControlPlane { + pub fn one_page_server(pgdata_base_path: String) -> TestStorageControlPlane { let env = local_env::test_env(); let pserver = Arc::new(PageServerNode { @@ -40,7 +40,15 @@ impl TestStorageControlPlane { listen_address: None, }); pserver.init(); - pserver.start().unwrap(); + + if pgdata_base_path.is_empty() + { + pserver.start().unwrap(); + } + else + { + pserver.start_fromdatadir(pgdata_base_path).unwrap(); + } TestStorageControlPlane { wal_acceptors: Vec::new(), @@ -142,7 +150,6 @@ impl PageServerNode { .args(&["-D", self.env.pageserver_data_dir().to_str().unwrap()]) .args(&["-l", self.address().to_string().as_str()]) .arg("-d") - .arg("--skip-recovery") .env_clear() .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) @@ -158,6 +165,30 @@ impl PageServerNode { } } + pub fn start_fromdatadir(&self, pgdata_base_path: String) -> Result<()> { + println!("Starting pageserver at '{}'", self.address()); + + let status = Command::new(self.env.zenith_distrib_dir.join("pageserver")) // XXX -> method + .args(&["-D", self.env.pageserver_data_dir().to_str().unwrap()]) + .args(&["-l", self.address().to_string().as_str()]) + .arg("-d") + .args(&["--restore-from", "local"]) + .env_clear() + .env("PATH", self.env.pg_bin_dir().to_str().unwrap()) // needs postres-wal-redo binary + .env("LD_LIBRARY_PATH", self.env.pg_lib_dir().to_str().unwrap()) + .env("PGDATA_BASE_PATH", pgdata_base_path) + .status()?; + + if !status.success() { + return Err(Box::::from(format!( + "Pageserver failed to start. See '{}' for details.", + self.env.pageserver_log().to_str().unwrap() + ))); + } else { + return Ok(()); + } + } + pub fn stop(&self) -> Result<()> { let pidfile = self.env.pageserver_pidfile(); let pid = read_pidfile(&pidfile)?; diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index 51f9ba9ddb..46853409d6 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -11,7 +11,7 @@ use control_plane::storage::TestStorageControlPlane; #[test] fn test_redo_cases() { // Start pageserver that reads WAL directly from that postgres - let storage_cplane = TestStorageControlPlane::one_page_server(); + let storage_cplane = TestStorageControlPlane::one_page_server(String::new()); let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // start postgres @@ -51,7 +51,7 @@ fn test_redo_cases() { #[ignore] fn test_regress() { // Start pageserver that reads WAL directly from that postgres - let storage_cplane = TestStorageControlPlane::one_page_server(); + let storage_cplane = TestStorageControlPlane::one_page_server(String::new()); let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // start postgres @@ -63,9 +63,10 @@ fn test_regress() { // Run two postgres instances on one pageserver #[test] +#[ignore] fn test_pageserver_multitenancy() { // Start pageserver that reads WAL directly from that postgres - let storage_cplane = TestStorageControlPlane::one_page_server(); + let storage_cplane = TestStorageControlPlane::one_page_server(String::new()); let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); // Allocate postgres instance, but don't start diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index ab8b78dd2d..177cfe4b24 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -34,3 +34,4 @@ postgres-protocol = { git = "https://github.com/kelvich/rust-postgres", branch = postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } anyhow = "1.0" crc32c = "0.6.0" +walkdir = "2" diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 2f34bb7a6f..f151e3773a 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -18,6 +18,7 @@ use slog_scope; use slog_stdlog; use pageserver::page_service; +use pageserver::restore_datadir; use pageserver::restore_s3; use pageserver::tui; use pageserver::walreceiver; @@ -51,10 +52,9 @@ fn main() -> Result<(), io::Error> { .long("daemonize") .takes_value(false) .help("Run in the background")) - .arg(Arg::with_name("skip_recovery") - .long("skip-recovery") - .takes_value(false) - .help("Skip S3 recovery procedy and start empty")) + .arg(Arg::with_name("restore-from") + .takes_value(true) + .help("Upload data from s3 or datadir")) .get_matches(); let mut conf = PageServerConf { @@ -63,7 +63,7 @@ fn main() -> Result<(), io::Error> { interactive: false, wal_producer_connstr: None, listen_addr: "127.0.0.1:5430".parse().unwrap(), - skip_recovery: false, + restore_from: String::new(), }; if let Some(dir) = arg_matches.value_of("datadir") { @@ -85,8 +85,8 @@ fn main() -> Result<(), io::Error> { )); } - if arg_matches.is_present("skip_recovery") { - conf.skip_recovery = true; + if let Some(restore_from) = arg_matches.value_of("restore_from") { + conf.restore_from = String::from(restore_from); } if let Some(addr) = arg_matches.value_of("wal_producer") { @@ -159,10 +159,13 @@ fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> { // Before opening up for connections, restore the latest base backup from S3. // (We don't persist anything to local disk at the moment, so we need to do // this at every startup) - if !conf.skip_recovery { + if conf.restore_from.eq("s3") { restore_s3::restore_main(&conf); + } else if conf.restore_from.eq("local") { + restore_datadir::restore_main(&conf); } + // Create directory for wal-redo datadirs match fs::create_dir(conf.data_dir.join("wal-redo")) { Ok(_) => {} diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index b504308e6b..8614e0b5ca 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; pub mod page_cache; pub mod page_service; +pub mod restore_datadir; pub mod restore_s3; pub mod tui; pub mod tui_event; @@ -19,5 +20,5 @@ pub struct PageServerConf { pub interactive: bool, pub wal_producer_connstr: Option, pub listen_addr: SocketAddr, - pub skip_recovery: bool, + pub restore_from: String, } diff --git a/pageserver/src/restore_datadir.rs b/pageserver/src/restore_datadir.rs new file mode 100644 index 0000000000..7c038493b2 --- /dev/null +++ b/pageserver/src/restore_datadir.rs @@ -0,0 +1,270 @@ +// +// Restore chunks from S3 +// +// This runs once at Page Server startup. It loads all the "base images" from +// S3 into the in-memory page cache. It also initializes the "last valid LSN" +// in the page cache to the LSN of the base image, so that when the WAL receiver +// is started, it starts streaming from that LSN. +// + +use bytes::{Buf, BytesMut}; +use log::*; +use regex::Regex; +use std::env; +use std::fmt; + +use tokio::runtime; + +use futures::future; + +use crate::{page_cache, PageServerConf}; +use std::fs; +use walkdir::WalkDir; + +pub fn restore_main(conf: &PageServerConf) { + // Create a new thread pool + let runtime = runtime::Runtime::new().unwrap(); + + runtime.block_on(async { + let result = restore_chunk(conf).await; + + match result { + Ok(_) => { + return; + } + Err(err) => { + error!("error: {}", err); + return; + } + } + }); +} + +async fn restore_chunk(conf: &PageServerConf) -> Result<(), FilePathError> { + let pgdata_base_path = env::var("PGDATA_BASE_PATH").unwrap(); + info!("Restoring from local dir..."); + + let sys_id: u64 = 42; + let control_lsn = 0; //TODO get it from sysid + let mut slurp_futures: Vec<_> = Vec::new(); + + for e in WalkDir::new(pgdata_base_path.clone()) { + let entry = e.unwrap(); + + if !entry.path().is_dir() { + let path = entry.path().to_str().unwrap(); + + let relpath = path + .strip_prefix(&format!("{}/", pgdata_base_path)) + .unwrap(); + info!( + "Restoring file {} relpath {}", + entry.path().display(), + relpath + ); + + let parsed = parse_rel_file_path(&relpath); + + match parsed { + Ok(mut p) => { + p.lsn = control_lsn; + + let f = slurp_base_file(conf, sys_id, path.to_string(), p); + + slurp_futures.push(f); + } + Err(e) => { + warn!("unrecognized file: {} ({})", relpath, e); + } + }; + } + } + + let pcache = page_cache::get_pagecache(conf.clone(), sys_id); + pcache.init_valid_lsn(control_lsn); + + info!("{} files to restore...", slurp_futures.len()); + + future::join_all(slurp_futures).await; + info!("restored!"); + Ok(()) +} + +// From pg_tablespace_d.h +// +// FIXME: we'll probably need these elsewhere too, move to some common location +const DEFAULTTABLESPACE_OID: u32 = 1663; +const GLOBALTABLESPACE_OID: u32 = 1664; + +#[derive(Debug)] +struct FilePathError { + msg: String, +} + +impl FilePathError { + fn new(msg: &str) -> FilePathError { + FilePathError { + msg: msg.to_string(), + } + } +} + +impl From for FilePathError { + fn from(e: core::num::ParseIntError) -> Self { + return FilePathError { + msg: format!("invalid filename: {}", e), + }; + } +} + +impl fmt::Display for FilePathError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "invalid filename") + } +} + +fn forkname_to_forknum(forkname: Option<&str>) -> Result { + match forkname { + // "main" is not in filenames, it's implicit if the fork name is not present + None => Ok(0), + Some("fsm") => Ok(1), + Some("vm") => Ok(2), + Some("init") => Ok(3), + Some(_) => Err(FilePathError::new("invalid forkname")), + } +} + +#[derive(Debug)] +struct ParsedBaseImageFileName { + pub spcnode: u32, + pub dbnode: u32, + pub relnode: u32, + pub forknum: u32, + pub segno: u32, + + pub lsn: u64, +} + +// formats: +// +// _ +// . +// _. +fn parse_filename(fname: &str) -> Result<(u32, u32, u32, u64), FilePathError> { + let re = Regex::new(r"^(?P\d+)(_(?P[a-z]+))?(\.(?P\d+))?$").unwrap(); + + let caps = re + .captures(fname) + .ok_or_else(|| FilePathError::new("invalid relation data file name"))?; + + let relnode_str = caps.name("relnode").unwrap().as_str(); + let relnode = u32::from_str_radix(relnode_str, 10)?; + + let forkname_match = caps.name("forkname"); + let forkname = if forkname_match.is_none() { + None + } else { + Some(forkname_match.unwrap().as_str()) + }; + let forknum = forkname_to_forknum(forkname)?; + + let segno_match = caps.name("segno"); + let segno = if segno_match.is_none() { + 0 + } else { + u32::from_str_radix(segno_match.unwrap().as_str(), 10)? + }; + return Ok((relnode, forknum, segno, 0)); +} + +fn parse_rel_file_path(path: &str) -> Result { + /* + * Relation data files can be in one of the following directories: + * + * global/ + * shared relations + * + * base// + * regular relations, default tablespace + * + * pg_tblspc/// + * within a non-default tablespace (the name of the directory + * depends on version) + * + * And the relation data files themselves have a filename like: + * + * . + */ + if let Some(fname) = path.strip_prefix("global/") { + let (relnode, forknum, segno, lsn) = parse_filename(fname)?; + + return Ok(ParsedBaseImageFileName { + spcnode: GLOBALTABLESPACE_OID, + dbnode: 0, + relnode, + forknum, + segno, + lsn, + }); + } else if let Some(dbpath) = path.strip_prefix("base/") { + let mut s = dbpath.split("/"); + let dbnode_str = s + .next() + .ok_or_else(|| FilePathError::new("invalid relation data file name"))?; + let dbnode = u32::from_str_radix(dbnode_str, 10)?; + let fname = s + .next() + .ok_or_else(|| FilePathError::new("invalid relation data file name"))?; + if s.next().is_some() { + return Err(FilePathError::new("invalid relation data file name")); + }; + + let (relnode, forknum, segno, lsn) = parse_filename(fname)?; + + return Ok(ParsedBaseImageFileName { + spcnode: DEFAULTTABLESPACE_OID, + dbnode, + relnode, + forknum, + segno, + lsn, + }); + } else if let Some(_) = path.strip_prefix("pg_tblspc/") { + // TODO + return Err(FilePathError::new("tablespaces not supported")); + } else { + return Err(FilePathError::new("invalid relation data file name")); + } +} + +async fn slurp_base_file( + conf: &PageServerConf, + sys_id: u64, + file_path: String, + parsed: ParsedBaseImageFileName, +) { + trace!("slurp_base_file local path {}", file_path); + + let data = fs::read(file_path).unwrap(); + let data_bytes: &[u8] = &data; + let mut bytes = BytesMut::from(data_bytes).freeze(); + + // FIXME: use constants (BLCKSZ) + let mut blknum: u32 = parsed.segno * (1024 * 1024 * 1024 / 8192); + + let pcache = page_cache::get_pagecache(conf.clone(), sys_id); + + while bytes.remaining() >= 8192 { + let tag = page_cache::BufferTag { + spcnode: parsed.spcnode, + dbnode: parsed.dbnode, + relnode: parsed.relnode, + forknum: parsed.forknum as u8, + blknum: blknum, + }; + + pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(8192)); + + blknum += 1; + } +} From 913a91c541532337f5b96213301c59b5a36f1822 Mon Sep 17 00:00:00 2001 From: anastasia Date: Thu, 15 Apr 2021 13:47:14 +0300 Subject: [PATCH 13/33] bump vendor/postgres --- vendor/postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/postgres b/vendor/postgres index b1f5a5ec14..ea5a673983 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit b1f5a5ec145d5d9614eec4824074edae1162e5fa +Subproject commit ea5a673983ac0d1e055b8da99d3dcca348cecc49 From 1190030872011bbee9aa9029b25ca3490c07f131 Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 14 Apr 2021 23:00:40 +0300 Subject: [PATCH 14/33] handle SLRU in restore_datadir --- control_plane/src/storage.rs | 7 +-- pageserver/src/bin/pageserver.rs | 1 - pageserver/src/lib.rs | 1 + pageserver/src/pg_constants.rs | 11 ++++ pageserver/src/restore_datadir.rs | 91 +++++++++++++++++++++++++++---- 5 files changed, 94 insertions(+), 17 deletions(-) create mode 100644 pageserver/src/pg_constants.rs diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 86c0d50327..b4608210d8 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -41,12 +41,9 @@ impl TestStorageControlPlane { }); pserver.init(); - if pgdata_base_path.is_empty() - { + if pgdata_base_path.is_empty() { pserver.start().unwrap(); - } - else - { + } else { pserver.start_fromdatadir(pgdata_base_path).unwrap(); } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index f151e3773a..353a667827 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -165,7 +165,6 @@ fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> { restore_datadir::restore_main(&conf); } - // Create directory for wal-redo datadirs match fs::create_dir(conf.data_dir.join("wal-redo")) { Ok(_) => {} diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 8614e0b5ca..9333fdae29 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; pub mod page_cache; pub mod page_service; +pub mod pg_constants; pub mod restore_datadir; pub mod restore_s3; pub mod tui; diff --git a/pageserver/src/pg_constants.rs b/pageserver/src/pg_constants.rs new file mode 100644 index 0000000000..b59ddb5396 --- /dev/null +++ b/pageserver/src/pg_constants.rs @@ -0,0 +1,11 @@ +// From pg_tablespace_d.h +// +pub const DEFAULTTABLESPACE_OID: u32 = 1663; +pub const GLOBALTABLESPACE_OID: u32 = 1664; +//Special values for non-rel files' tags +//TODO maybe use enum? +pub const PG_CONTROLFILE_FORKNUM: u32 = 42; +pub const PG_FILENODEMAP_FORKNUM: u32 = 43; +pub const PG_XACT_FORKNUM: u32 = 44; +pub const PG_MXACT_OFFSETS_FORKNUM: u32 = 45; +pub const PG_MXACT_MEMBERS_FORKNUM: u32 = 46; diff --git a/pageserver/src/restore_datadir.rs b/pageserver/src/restore_datadir.rs index 7c038493b2..98a9de56f2 100644 --- a/pageserver/src/restore_datadir.rs +++ b/pageserver/src/restore_datadir.rs @@ -17,7 +17,7 @@ use tokio::runtime; use futures::future; -use crate::{page_cache, PageServerConf}; +use crate::{page_cache, pg_constants, PageServerConf}; use std::fs; use walkdir::WalkDir; @@ -90,12 +90,6 @@ async fn restore_chunk(conf: &PageServerConf) -> Result<(), FilePathError> { Ok(()) } -// From pg_tablespace_d.h -// -// FIXME: we'll probably need these elsewhere too, move to some common location -const DEFAULTTABLESPACE_OID: u32 = 1663; -const GLOBALTABLESPACE_OID: u32 = 1664; - #[derive(Debug)] struct FilePathError { msg: String, @@ -196,10 +190,32 @@ fn parse_rel_file_path(path: &str) -> Result. */ if let Some(fname) = path.strip_prefix("global/") { + if fname.contains("pg_control") { + return Ok(ParsedBaseImageFileName { + spcnode: pg_constants::GLOBALTABLESPACE_OID, + dbnode: 0, + relnode: 0, + forknum: pg_constants::PG_CONTROLFILE_FORKNUM, + segno: 0, + lsn: 0, + }); + } + + if fname.contains("pg_filenode") { + return Ok(ParsedBaseImageFileName { + spcnode: pg_constants::GLOBALTABLESPACE_OID, + dbnode: 0, + relnode: 0, + forknum: pg_constants::PG_FILENODEMAP_FORKNUM, + segno: 0, + lsn: 0, + }); + } + let (relnode, forknum, segno, lsn) = parse_filename(fname)?; return Ok(ParsedBaseImageFileName { - spcnode: GLOBALTABLESPACE_OID, + spcnode: pg_constants::GLOBALTABLESPACE_OID, dbnode: 0, relnode, forknum, @@ -219,16 +235,54 @@ fn parse_rel_file_path(path: &str) -> Result= 8192 { let tag = page_cache::BufferTag { spcnode: parsed.spcnode, @@ -265,6 +333,7 @@ async fn slurp_base_file( pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(8192)); + pcache.relsize_inc(&reltag, Some(blknum)); blknum += 1; } } From d7eeaec70608917b1c91c5800aa4bbc8630afc3b Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 14 Apr 2021 19:47:05 +0300 Subject: [PATCH 15/33] add test for restore from local pgdata --- control_plane/src/compute.rs | 2 +- control_plane/src/storage.rs | 17 ++++++++++ integration_tests/tests/test_pageserver.rs | 36 ++++++++++++++++++++++ pageserver/src/bin/pageserver.rs | 7 +++-- 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 205a1889e2..30850dbd52 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -260,7 +260,7 @@ impl PostgresNode { Ok(()) } - fn pgdata(&self) -> PathBuf { + pub fn pgdata(&self) -> PathBuf { self.env.compute_dir().join(self.name.clone()) } diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index b4608210d8..394f9b7aac 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -54,6 +54,23 @@ impl TestStorageControlPlane { } } + pub fn one_page_server_no_start() -> TestStorageControlPlane { + let env = local_env::test_env(); + + let pserver = Arc::new(PageServerNode { + env: env.clone(), + kill_on_exit: true, + listen_address: None, + }); + pserver.init(); + + TestStorageControlPlane { + wal_acceptors: Vec::new(), + pageserver: pserver, + test_done: AtomicBool::new(false), + } + } + // postgres <-> {wal_acceptor1, wal_acceptor2, ...} pub fn fault_tolerant(redundancy: usize) -> TestStorageControlPlane { let env = local_env::test_env(); diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index 46853409d6..55c0cbc6df 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -3,6 +3,9 @@ use control_plane::compute::ComputeControlPlane; use control_plane::storage::TestStorageControlPlane; +use std::thread::sleep; +use std::time::Duration; + // XXX: force all redo at the end // -- restart + seqscan won't read deleted stuff // -- pageserver api endpoint to check all rels @@ -109,3 +112,36 @@ fn test_pageserver_multitenancy() { println!("sum = {}", count); assert_eq!(count, 15000150000); } + +#[test] +fn test_upload_pageserver_local() { + // Init pageserver that reads WAL directly from that postgres + // Don't start yet + + let storage_cplane = TestStorageControlPlane::one_page_server_no_start(); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); + + // init postgres node + let node = compute_cplane.new_test_node(); + + //upload data to pageserver & start it + &storage_cplane + .pageserver + .start_fromdatadir(node.pgdata().to_str().unwrap().to_string()) + .unwrap(); + + sleep(Duration::from_secs(10)); + + // start postgres node + node.start().unwrap(); + + // check basic work with table + node.safe_psql( + "postgres", + "CREATE TABLE t(key int primary key, value text)", + ); + node.safe_psql( + "postgres", + "INSERT INTO t SELECT generate_series(1,100000), 'payload'", + ); +} diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 353a667827..6d051fcfc9 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -52,7 +52,8 @@ fn main() -> Result<(), io::Error> { .long("daemonize") .takes_value(false) .help("Run in the background")) - .arg(Arg::with_name("restore-from") + .arg(Arg::with_name("restore_from") + .long("restore-from") .takes_value(true) .help("Upload data from s3 or datadir")) .get_matches(); @@ -154,14 +155,16 @@ fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> { let mut threads = Vec::new(); - info!("starting..."); + info!("starting... {}", conf.restore_from); // Before opening up for connections, restore the latest base backup from S3. // (We don't persist anything to local disk at the moment, so we need to do // this at every startup) if conf.restore_from.eq("s3") { + info!("restore-from s3..."); restore_s3::restore_main(&conf); } else if conf.restore_from.eq("local") { + info!("restore-from local..."); restore_datadir::restore_main(&conf); } From 05886b33e5e962e2cb25478630b59121898b2efa Mon Sep 17 00:00:00 2001 From: anastasia Date: Wed, 14 Apr 2021 23:16:16 +0300 Subject: [PATCH 16/33] fix typo --- pageserver/src/restore_datadir.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pageserver/src/restore_datadir.rs b/pageserver/src/restore_datadir.rs index 98a9de56f2..d490e25dea 100644 --- a/pageserver/src/restore_datadir.rs +++ b/pageserver/src/restore_datadir.rs @@ -265,7 +265,7 @@ fn parse_rel_file_path(path: &str) -> Result Result Date: Thu, 15 Apr 2021 13:39:46 +0300 Subject: [PATCH 17/33] don't ignore multitenancy test --- integration_tests/tests/test_pageserver.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index 55c0cbc6df..ea50524c96 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -66,7 +66,6 @@ fn test_regress() { // Run two postgres instances on one pageserver #[test] -#[ignore] fn test_pageserver_multitenancy() { // Start pageserver that reads WAL directly from that postgres let storage_cplane = TestStorageControlPlane::one_page_server(String::new()); From 24c3e961e4a2dbcd812a1c61e45d5a8b850de72c Mon Sep 17 00:00:00 2001 From: anastasia Date: Thu, 15 Apr 2021 16:33:46 +0300 Subject: [PATCH 18/33] Always run cargo build before tests in CI --- .github/workflows/testing.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2f8acc4385..450b93d85a 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -77,10 +77,7 @@ jobs: target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - # That build is only to build dependencies and can be skipped if Cargo.lock - # wasn't changed. Next steps need their own build - - name: Install cargo deps - if: steps.cache_cargo.outputs.cache-hit != 'true' + - name: Build run: | cargo build From b4c5cb27731340f66c19ce87b37e9e7c27675972 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 15 Apr 2021 16:17:22 +0300 Subject: [PATCH 19/33] Clean up error types a little bit. Don't use std::io::Error for errors that are not I/O related. Prefer anyhow::Result instead. --- pageserver/src/bin/pageserver.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 6d051fcfc9..af2fb8c2f7 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -6,11 +6,13 @@ use log::*; use std::fs; use std::io; use std::path::PathBuf; +use std::process::exit; use std::thread; use std::{fs::File, fs::OpenOptions, str::FromStr}; use clap::{App, Arg}; use daemonize::Daemonize; +use anyhow::Result; use slog; use slog::Drain; @@ -24,7 +26,7 @@ use pageserver::tui; use pageserver::walreceiver; use pageserver::PageServerConf; -fn main() -> Result<(), io::Error> { +fn main() -> Result<()> { let arg_matches = App::new("Zenith page server") .about("Materializes WAL stream to pages and serves them to the postgres") .arg(Arg::with_name("datadir") @@ -80,10 +82,8 @@ fn main() -> Result<(), io::Error> { } if conf.daemonize && conf.interactive { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "--daemonize is not allowed with --interactive: choose one", - )); + eprintln!("--daemonize is not allowed with --interactive: choose one"); + exit(1); } if let Some(restore_from) = arg_matches.value_of("restore_from") { @@ -101,7 +101,7 @@ fn main() -> Result<(), io::Error> { start_pageserver(conf) } -fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> { +fn start_pageserver(conf: PageServerConf) -> Result<()> { // Initialize logger let _scope_guard = init_logging(&conf)?; let _log_guard = slog_stdlog::init().unwrap(); @@ -174,7 +174,7 @@ fn start_pageserver(conf: PageServerConf) -> Result<(), io::Error> { Err(e) => match e.kind() { io::ErrorKind::AlreadyExists => {} _ => { - panic!("Failed to create wal-redo data directory: {}", e); + anyhow::bail!("Failed to create wal-redo data directory: {}", e); } }, } From d2c3ad162a879969b98e7e498526821b1bfc03cd Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 15 Apr 2021 15:58:53 +0300 Subject: [PATCH 20/33] Prefer passing PageServerConf by reference. It seems more idiomatic Rust. --- pageserver/src/bin/pageserver.rs | 18 +++++++++--------- pageserver/src/page_cache.rs | 5 +++-- pageserver/src/page_service.rs | 6 +++--- pageserver/src/restore_datadir.rs | 4 ++-- pageserver/src/restore_s3.rs | 4 ++-- pageserver/src/walreceiver.rs | 10 +++++----- pageserver/src/walredo.rs | 4 ++-- 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index af2fb8c2f7..8bbe31fdd7 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -98,10 +98,10 @@ fn main() -> Result<()> { conf.listen_addr = addr.parse().unwrap(); } - start_pageserver(conf) + start_pageserver(&conf) } -fn start_pageserver(conf: PageServerConf) -> Result<()> { +fn start_pageserver(conf: &PageServerConf) -> Result<()> { // Initialize logger let _scope_guard = init_logging(&conf)?; let _log_guard = slog_stdlog::init().unwrap(); @@ -186,13 +186,13 @@ fn start_pageserver(conf: PageServerConf) -> Result<()> { // // All other wal receivers are started on demand by "callmemaybe" command // sent to pageserver. - let conf_copy = conf.clone(); - if let Some(wal_producer) = conf.wal_producer_connstr { - let conf = conf_copy.clone(); + if let Some(wal_producer) = &conf.wal_producer_connstr { + let conf_copy = conf.clone(); + let wal_producer = wal_producer.clone(); let walreceiver_thread = thread::Builder::new() .name("static WAL receiver thread".into()) .spawn(move || { - walreceiver::thread_main(conf, &wal_producer); + walreceiver::thread_main(&conf_copy, &wal_producer); }) .unwrap(); threads.push(walreceiver_thread); @@ -200,12 +200,12 @@ fn start_pageserver(conf: PageServerConf) -> Result<()> { // GetPage@LSN requests are served by another thread. (It uses async I/O, // but the code in page_service sets up it own thread pool for that) - let conf = conf_copy.clone(); + let conf_copy = conf.clone(); let page_server_thread = thread::Builder::new() .name("Page Service thread".into()) - .spawn(|| { + .spawn(move || { // thread code - page_service::thread_main(conf); + page_service::thread_main(&conf_copy); }) .unwrap(); threads.push(page_server_thread); diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index c514292340..a453c10a73 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -113,7 +113,7 @@ lazy_static! { pub static ref PAGECACHES: Mutex>> = Mutex::new(HashMap::new()); } -pub fn get_pagecache(conf: PageServerConf, sys_id: u64) -> Arc { +pub fn get_pagecache(conf: &PageServerConf, sys_id: u64) -> Arc { let mut pcaches = PAGECACHES.lock().unwrap(); if !pcaches.contains_key(&sys_id) { @@ -124,10 +124,11 @@ pub fn get_pagecache(conf: PageServerConf, sys_id: u64) -> Arc { // Now join_handle is not saved any where and we won'try restart tharead // if it is dead. We may later stop that treads after some inactivity period // and restart them on demand. + let conf = conf.clone(); let _walredo_thread = thread::Builder::new() .name("WAL redo thread".into()) .spawn(move || { - walredo::wal_redo_main(conf, sys_id); + walredo::wal_redo_main(&conf, sys_id); }) .unwrap(); } diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index fcaf8c11f6..1feb4d6f12 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -215,7 +215,7 @@ impl FeMessage { /////////////////////////////////////////////////////////////////////////////// -pub fn thread_main(conf: PageServerConf) { +pub fn thread_main(conf: &PageServerConf) { // Create a new thread pool // // FIXME: keep it single-threaded for now, make it easier to debug with gdb, @@ -458,7 +458,7 @@ impl Connection { let _walreceiver_thread = thread::Builder::new() .name("WAL receiver thread".into()) .spawn(move || { - walreceiver::thread_main(conf_copy, &connstr); + walreceiver::thread_main(&conf_copy, &connstr); }) .unwrap(); @@ -503,7 +503,7 @@ impl Connection { self.stream.write_i16(0).await?; /* numAttributes */ self.stream.flush().await?; - let pcache = page_cache::get_pagecache(self.conf.clone(), sysid); + let pcache = page_cache::get_pagecache(&self.conf, sysid); loop { let message = self.read_message().await?; diff --git a/pageserver/src/restore_datadir.rs b/pageserver/src/restore_datadir.rs index d490e25dea..985f5e3905 100644 --- a/pageserver/src/restore_datadir.rs +++ b/pageserver/src/restore_datadir.rs @@ -80,7 +80,7 @@ async fn restore_chunk(conf: &PageServerConf) -> Result<(), FilePathError> { } } - let pcache = page_cache::get_pagecache(conf.clone(), sys_id); + let pcache = page_cache::get_pagecache(conf, sys_id); pcache.init_valid_lsn(control_lsn); info!("{} files to restore...", slurp_futures.len()); @@ -313,7 +313,7 @@ async fn slurp_base_file( // FIXME: use constants (BLCKSZ) let mut blknum: u32 = parsed.segno * (1024 * 1024 * 1024 / 8192); - let pcache = page_cache::get_pagecache(conf.clone(), sys_id); + let pcache = page_cache::get_pagecache(conf, sys_id); let reltag = page_cache::RelTag { spcnode: parsed.spcnode, diff --git a/pageserver/src/restore_s3.rs b/pageserver/src/restore_s3.rs index 08ba3e7fa3..17e045fb5b 100644 --- a/pageserver/src/restore_s3.rs +++ b/pageserver/src/restore_s3.rs @@ -119,7 +119,7 @@ async fn restore_chunk(conf: &PageServerConf) -> Result<(), S3Error> { panic!("no base backup found"); } - let pcache = page_cache::get_pagecache(conf.clone(), sys_id); + let pcache = page_cache::get_pagecache(conf, sys_id); pcache.init_valid_lsn(oldest_lsn); info!("{} files to restore...", slurp_futures.len()); @@ -305,7 +305,7 @@ async fn slurp_base_file( // FIXME: use constants (BLCKSZ) let mut blknum: u32 = parsed.segno * (1024 * 1024 * 1024 / 8192); - let pcache = page_cache::get_pagecache(conf.clone(), sys_id); + let pcache = page_cache::get_pagecache(conf, sys_id); while bytes.remaining() >= 8192 { let tag = page_cache::BufferTag { diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 9f382b2efb..7ce3e9727f 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -22,7 +22,7 @@ use tokio_postgres::{connect_replication, Error, NoTls, ReplicationMode}; // // This is the entry point for the WAL receiver thread. // -pub fn thread_main(conf: PageServerConf, wal_producer_connstr: &String) { +pub fn thread_main(conf: &PageServerConf, wal_producer_connstr: &str) { info!("WAL receiver thread started: '{}'", wal_producer_connstr); let runtime = runtime::Builder::new_current_thread() @@ -32,7 +32,7 @@ pub fn thread_main(conf: PageServerConf, wal_producer_connstr: &String) { runtime.block_on(async { loop { - let _res = walreceiver_main(conf.clone(), wal_producer_connstr).await; + let _res = walreceiver_main(conf, wal_producer_connstr).await; // TODO: print/log the error info!( @@ -45,13 +45,13 @@ pub fn thread_main(conf: PageServerConf, wal_producer_connstr: &String) { } async fn walreceiver_main( - conf: PageServerConf, - wal_producer_connstr: &String, + conf: &PageServerConf, + wal_producer_connstr: &str, ) -> Result<(), Error> { // Connect to the database in replication mode. debug!("connecting to {}...", wal_producer_connstr); let (mut rclient, connection) = connect_replication( - wal_producer_connstr.as_str(), + wal_producer_connstr, NoTls, ReplicationMode::Physical, ) diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 685e771f4a..9d5a1ee143 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -41,7 +41,7 @@ static TIMEOUT: Duration = Duration::from_secs(20); // // Main entry point for the WAL applicator thread. // -pub fn wal_redo_main(conf: PageServerConf, sys_id: u64) { +pub fn wal_redo_main(conf: &PageServerConf, sys_id: u64) { info!("WAL redo thread started {}", sys_id); // We block on waiting for requests on the walredo request channel, but @@ -52,7 +52,7 @@ pub fn wal_redo_main(conf: PageServerConf, sys_id: u64) { .build() .unwrap(); - let pcache = page_cache::get_pagecache(conf.clone(), sys_id); + let pcache = page_cache::get_pagecache(conf, sys_id); // Loop forever, handling requests as they come. let walredo_channel_receiver = &pcache.walredo_receiver; From e8032f26e6ed948601dce930d08e28f47605bc48 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Thu, 15 Apr 2021 10:43:23 -0700 Subject: [PATCH 21/33] adopt new tokio-postgres:replication branch This PR has evolved a lot; jump to the newer version. This should make it easier to handle keepalive messages. --- Cargo.lock | 36 +++++-------- control_plane/Cargo.toml | 4 +- integration_tests/Cargo.toml | 4 +- pageserver/Cargo.toml | 7 +-- pageserver/src/walreceiver.rs | 95 +++++++++++++++++++++++++++-------- walkeeper/Cargo.toml | 6 +-- 6 files changed, 97 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe4be36d72..8299cbe7d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -872,7 +872,7 @@ dependencies = [ "httpdate", "itoa", "pin-project", - "socket2 0.4.0", + "socket2", "tokio", "tower-service", "tracing", @@ -1089,7 +1089,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a19900e7eee95eb2b3c2e26d12a874cc80aaf750e31be6fcbe743ead369fa45d" dependencies = [ "libc", - "socket2 0.4.0", + "socket2", ] [[package]] @@ -1209,6 +1209,7 @@ dependencies = [ "log", "postgres", "postgres-protocol", + "postgres-types", "rand 0.8.3", "regex", "rust-s3", @@ -1333,8 +1334,8 @@ dependencies = [ [[package]] name = "postgres" -version = "0.19.0" -source = "git+https://github.com/kelvich/rust-postgres?branch=replication_rebase#f3425d991f75cb7b464a37e6b3d5d05f8bf51c02" +version = "0.19.1" +source = "git+https://github.com/zenithdb/rust-postgres.git?rev=a0d067b66447951d1276a53fb09886539c3fa094#a0d067b66447951d1276a53fb09886539c3fa094" dependencies = [ "bytes", "fallible-iterator", @@ -1346,8 +1347,8 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.0" -source = "git+https://github.com/kelvich/rust-postgres?branch=replication_rebase#f3425d991f75cb7b464a37e6b3d5d05f8bf51c02" +version = "0.6.1" +source = "git+https://github.com/zenithdb/rust-postgres.git?rev=a0d067b66447951d1276a53fb09886539c3fa094#a0d067b66447951d1276a53fb09886539c3fa094" dependencies = [ "base64", "byteorder", @@ -1363,8 +1364,8 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.0" -source = "git+https://github.com/kelvich/rust-postgres?branch=replication_rebase#f3425d991f75cb7b464a37e6b3d5d05f8bf51c02" +version = "0.2.1" +source = "git+https://github.com/zenithdb/rust-postgres.git?rev=a0d067b66447951d1276a53fb09886539c3fa094#a0d067b66447951d1276a53fb09886539c3fa094" dependencies = [ "bytes", "fallible-iterator", @@ -1882,17 +1883,6 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" -[[package]] -name = "socket2" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" -dependencies = [ - "cfg-if 1.0.0", - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.4.0" @@ -2086,8 +2076,8 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.0" -source = "git+https://github.com/kelvich/rust-postgres?branch=replication_rebase#f3425d991f75cb7b464a37e6b3d5d05f8bf51c02" +version = "0.7.1" +source = "git+https://github.com/zenithdb/rust-postgres.git?rev=a0d067b66447951d1276a53fb09886539c3fa094#a0d067b66447951d1276a53fb09886539c3fa094" dependencies = [ "async-trait", "byteorder", @@ -2098,10 +2088,10 @@ dependencies = [ "parking_lot", "percent-encoding", "phf", - "pin-project", + "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2 0.3.19", + "socket2", "tokio", "tokio-util", ] diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 4699a4da40..c4ec2e9b33 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -8,8 +8,8 @@ edition = "2018" [dependencies] rand = "0.8.3" -postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } -tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } +postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } +tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } serde = "" serde_derive = "" diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index ad7913cc5c..51f9d0c773 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -9,8 +9,8 @@ edition = "2018" [dependencies] lazy_static = "1.4.0" rand = "0.8.3" -postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } -tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } +postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } +tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } pageserver = { path = "../pageserver" } walkeeper = { path = "../walkeeper" } diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 177cfe4b24..5da883d81a 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -29,9 +29,10 @@ daemonize = "0.4.1" rust-s3 = { git = "https://github.com/hlinnaka/rust-s3", features = ["no-verify-ssl"] } tokio = { version = "1.3.0", features = ["full"] } tokio-stream = { version = "0.1.4" } -tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } -postgres-protocol = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } -postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } +tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } +postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } +postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } +postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } anyhow = "1.0" crc32c = "0.6.0" walkdir = "2" diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 7ce3e9727f..76471084f7 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + // // WAL receiver // @@ -13,11 +15,14 @@ use tokio_stream::StreamExt; use crate::page_cache; use crate::page_cache::BufferTag; -use crate::waldecoder::WalStreamDecoder; +use crate::waldecoder::{decode_wal_record, WalStreamDecoder}; use crate::PageServerConf; use postgres_protocol::message::backend::ReplicationMessage; -use tokio_postgres::{connect_replication, Error, NoTls, ReplicationMode}; +use postgres_types::PgLsn; +use tokio_postgres::replication::ReplicationStream; + +use tokio_postgres::{Error, NoTls, SimpleQueryMessage, SimpleQueryRow}; // // This is the entry point for the WAL receiver thread. @@ -49,14 +54,10 @@ async fn walreceiver_main( wal_producer_connstr: &str, ) -> Result<(), Error> { // Connect to the database in replication mode. - debug!("connecting to {}...", wal_producer_connstr); - let (mut rclient, connection) = connect_replication( - wal_producer_connstr, - NoTls, - ReplicationMode::Physical, - ) - .await?; - debug!("connected!"); + info!("connecting to {:?}", wal_producer_connstr); + let connect_cfg = format!("{} replication=true", wal_producer_connstr); + let (rclient, connection) = tokio_postgres::connect(&connect_cfg, NoTls).await?; + info!("connected!"); // The connection object performs the actual communication with the database, // so spawn it off to run on its own. @@ -66,21 +67,21 @@ async fn walreceiver_main( } }); - let identify_system = rclient.identify_system().await?; - let end_of_wal = u64::from(identify_system.xlogpos()); + let identify = identify_system(&rclient).await?; + info!("{:?}", identify); + let end_of_wal = u64::from(identify.xlogpos); let mut caught_up = false; - let sysid: u64 = identify_system.systemid().parse().unwrap(); - let pcache = page_cache::get_pagecache(conf, sysid); + let pcache = page_cache::get_pagecache(conf, identify.systemid); // // Start streaming the WAL, from where we left off previously. // let mut startpoint = pcache.get_last_valid_lsn(); if startpoint == 0 { - // If we start here with identify_system.xlogpos() we will have race condition with + // If we start here with identify.xlogpos we will have race condition with // postgres start: insert into postgres may request page that was modified with lsn - // smaller than identify_system.xlogpos(). + // smaller than identify.xlogpos. // // Current procedure for starting postgres will anyway be changed to something // different like having 'initdb' method on a pageserver (or importing some shared @@ -105,10 +106,17 @@ async fn walreceiver_main( (end_of_wal >> 32), (end_of_wal & 0xffffffff) ); - let startpoint = tokio_postgres::types::Lsn::from(startpoint); - let mut physical_stream = rclient - .start_physical_replication(None, startpoint, None) - .await?; + + let startpoint = PgLsn::from(startpoint); + let query = format!("START_REPLICATION PHYSICAL {}", startpoint); + let copy_stream = rclient + .copy_both_simple::(&query) + .await + .unwrap(); + + let physical_stream = ReplicationStream::new(copy_stream); + tokio::pin!(physical_stream); + let mut waldecoder = WalStreamDecoder::new(u64::from(startpoint)); while let Some(replication_message) = physical_stream.next().await { @@ -132,8 +140,7 @@ async fn walreceiver_main( loop { if let Some((lsn, recdata)) = waldecoder.poll_decode() { - let decoded = - crate::waldecoder::decode_wal_record(startlsn, recdata.clone()); + let decoded = decode_wal_record(startlsn, recdata.clone()); // Put the WAL record to the page cache. We make a separate copy of // it for every block it modifies. (The actual WAL record is kept in @@ -192,3 +199,47 @@ async fn walreceiver_main( } return Ok(()); } + +/// Data returned from the postgres `IDENTIFY_SYSTEM` command +/// +/// See the [postgres docs] for more details. +/// +/// [postgres docs]: https://www.postgresql.org/docs/current/protocol-replication.html +#[derive(Debug)] +pub struct IdentifySystem { + systemid: u64, + timeline: u32, + xlogpos: PgLsn, + dbname: Option, +} + +/// Run the postgres `IDENTIFY_SYSTEM` command +pub async fn identify_system(client: &tokio_postgres::Client) -> Result { + let query_str = "IDENTIFY_SYSTEM"; + let response = client.simple_query(query_str).await?; + + // get(N) from row, then parse it as some destination type. + fn get_parse(row: &SimpleQueryRow, idx: usize) -> Option + where + T: FromStr, + { + let val = row.get(idx)?; + val.parse::().ok() + } + + // FIXME: turn unwrap() into errors. + // All of the tokio_postgres::Error builders are private, so we + // can't create them here. We'll just have to create our own error type. + + if let SimpleQueryMessage::Row(first_row) = response.get(0).unwrap() { + Ok(IdentifySystem { + systemid: get_parse(first_row, 0).unwrap(), + timeline: get_parse(first_row, 1).unwrap(), + xlogpos: get_parse(first_row, 2).unwrap(), + dbname: get_parse(first_row, 3), + }) + } else { + // FIXME: return an error + panic!("identify_system returned non-row response"); + } +} diff --git a/walkeeper/Cargo.toml b/walkeeper/Cargo.toml index 76dcd12582..e025a26d9b 100644 --- a/walkeeper/Cargo.toml +++ b/walkeeper/Cargo.toml @@ -29,9 +29,9 @@ daemonize = "0.4.1" rust-s3 = { git = "https://github.com/hlinnaka/rust-s3", features = ["no-verify-ssl"] } tokio = { version = "1.3.0", features = ["full"] } tokio-stream = { version = "0.1.4" } -tokio-postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } -postgres-protocol = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } -postgres = { git = "https://github.com/kelvich/rust-postgres", branch = "replication_rebase" } +tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } +postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } +postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } anyhow = "1.0" crc32c = "0.6.0" From 2246b4834883c236e93ad4132f509a403709f213 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Thu, 15 Apr 2021 14:28:47 -0700 Subject: [PATCH 22/33] handle keepalive messages When postgres sends us a keepalive message, send a reply so it doesn't time out and close the connection. The LSN values sent may need to change in the future. Currently we send: write_lsn <= page_cache last_valid_lsn flush_lsn <= page_cache last_valid_lsn apply_lsn <= 0 --- pageserver/src/walreceiver.rs | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 76471084f7..ecd5cd6ae8 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -20,7 +20,7 @@ use crate::PageServerConf; use postgres_protocol::message::backend::ReplicationMessage; use postgres_types::PgLsn; -use tokio_postgres::replication::ReplicationStream; +use tokio_postgres::replication::{PgTimestamp, ReplicationStream}; use tokio_postgres::{Error, NoTls, SimpleQueryMessage, SimpleQueryRow}; @@ -190,9 +190,32 @@ async fn walreceiver_main( } } - ReplicationMessage::PrimaryKeepAlive(_keepalive) => { - trace!("received PrimaryKeepAlive"); - // FIXME: Reply, or the connection will time out + ReplicationMessage::PrimaryKeepAlive(keepalive) => { + let wal_end = keepalive.wal_end(); + let timestamp = keepalive.timestamp(); + let reply_requested: bool = keepalive.reply() != 0; + + trace!( + "received PrimaryKeepAlive(wal_end: {}, timestamp: {} reply: {})", + wal_end, + timestamp, + reply_requested, + ); + if reply_requested { + // TODO: More thought should go into what values are sent here. + let last_lsn = PgLsn::from(pcache.get_last_valid_lsn()); + let write_lsn = last_lsn; + let flush_lsn = last_lsn; + let apply_lsn = PgLsn::INVALID; + let ts = PgTimestamp::now().unwrap(); + const NO_REPLY: u8 = 0u8; + + physical_stream + .as_mut() + .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY) + .await + .unwrap(); + } } _ => (), } From 4ff248515bd80365e3e0b032bb3fb4de05a7fe35 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Fri, 16 Apr 2021 13:33:23 -0700 Subject: [PATCH 23/33] remote unnecessary dependencies between peer crates These dependencies make cargo rebuild more than is strictly necessary. Removing them makes the build a little faster. --- Cargo.lock | 5 ----- control_plane/Cargo.toml | 3 --- integration_tests/Cargo.toml | 2 -- walkeeper/Cargo.toml | 2 -- 4 files changed, 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8299cbe7d9..2e818f3f4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -384,7 +384,6 @@ version = "0.1.0" dependencies = [ "home", "lazy_static", - "pageserver", "postgres", "rand 0.8.3", "regex", @@ -392,7 +391,6 @@ dependencies = [ "serde_derive", "tokio-postgres", "toml", - "walkeeper", ] [[package]] @@ -928,11 +926,9 @@ version = "0.1.0" dependencies = [ "control_plane", "lazy_static", - "pageserver", "postgres", "rand 0.8.3", "tokio-postgres", - "walkeeper", ] [[package]] @@ -2295,7 +2291,6 @@ dependencies = [ "futures", "lazy_static", "log", - "pageserver", "postgres", "postgres-protocol", "rand 0.8.3", diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index c4ec2e9b33..7281595c18 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -17,6 +17,3 @@ toml = "" home = "0.5.3" lazy_static = "" regex = "1" - -pageserver = { path = "../pageserver" } -walkeeper = { path = "../walkeeper" } diff --git a/integration_tests/Cargo.toml b/integration_tests/Cargo.toml index 51f9d0c773..b201b1849e 100644 --- a/integration_tests/Cargo.toml +++ b/integration_tests/Cargo.toml @@ -12,6 +12,4 @@ rand = "0.8.3" postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } -pageserver = { path = "../pageserver" } -walkeeper = { path = "../walkeeper" } control_plane = { path = "../control_plane" } diff --git a/walkeeper/Cargo.toml b/walkeeper/Cargo.toml index e025a26d9b..21e49bb096 100644 --- a/walkeeper/Cargo.toml +++ b/walkeeper/Cargo.toml @@ -34,5 +34,3 @@ postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } anyhow = "1.0" crc32c = "0.6.0" - -pageserver = { path = "../pageserver" } From 35e0099ac628e1328368d1ea211deeb91afccd63 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Fri, 16 Apr 2021 13:47:19 -0700 Subject: [PATCH 24/33] pin remote rust-s3 dependency to a git hash Using the hash should allow us to change the remote repo and propagate that change to user builds without that change becoming visible at a random time. It's unfortunate that we can't declare this dependency once in the top-level Cargo.toml; that feature request is rust-lang rfc 2906. --- Cargo.lock | 2 +- pageserver/Cargo.toml | 2 +- walkeeper/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e818f3f4b..74b32d20e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1616,7 +1616,7 @@ dependencies = [ [[package]] name = "rust-s3" version = "0.27.0-beta1" -source = "git+https://github.com/hlinnaka/rust-s3#7f15a24ec7daa0a5d9516da706212745f9042818" +source = "git+https://github.com/hlinnaka/rust-s3?rev=7f15a24ec7daa0a5d9516da706212745f9042818#7f15a24ec7daa0a5d9516da706212745f9042818" dependencies = [ "async-std", "async-trait", diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 5da883d81a..7f0b31e20c 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -26,7 +26,7 @@ clap = "2.33.0" termion = "1.5.6" tui = "0.14.0" daemonize = "0.4.1" -rust-s3 = { git = "https://github.com/hlinnaka/rust-s3", features = ["no-verify-ssl"] } +rust-s3 = { git = "https://github.com/hlinnaka/rust-s3", rev="7f15a24ec7daa0a5d9516da706212745f9042818", features = ["no-verify-ssl"] } tokio = { version = "1.3.0", features = ["full"] } tokio-stream = { version = "0.1.4" } tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } diff --git a/walkeeper/Cargo.toml b/walkeeper/Cargo.toml index 21e49bb096..98c63c434f 100644 --- a/walkeeper/Cargo.toml +++ b/walkeeper/Cargo.toml @@ -26,7 +26,7 @@ clap = "2.33.0" termion = "1.5.6" tui = "0.14.0" daemonize = "0.4.1" -rust-s3 = { git = "https://github.com/hlinnaka/rust-s3", features = ["no-verify-ssl"] } +rust-s3 = { git = "https://github.com/hlinnaka/rust-s3", rev="7f15a24ec7daa0a5d9516da706212745f9042818", features = ["no-verify-ssl"] } tokio = { version = "1.3.0", features = ["full"] } tokio-stream = { version = "0.1.4" } tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b66447951d1276a53fb09886539c3fa094" } From 639c9e826659b44b2a42f0a8bc7061fcc87cdf2e Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Thu, 15 Apr 2021 17:25:09 -0700 Subject: [PATCH 25/33] .gitignore vscode files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7d9bf1115c..768b75b413 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /tmp_check /tmp_install /tmp_check_cli +.vscode From 52d62758121c1523b3b1356344a2ac655c918591 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Mon, 12 Apr 2021 14:58:55 -0700 Subject: [PATCH 26/33] drop nonfunctional attributes allow(dead_code) These had no effect, so remove them. --- integration_tests/tests/test_pageserver.rs | 1 - pageserver/src/lib.rs | 1 - pageserver/src/tui_event.rs | 1 - walkeeper/src/lib.rs | 1 - 4 files changed, 4 deletions(-) diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index ea50524c96..c0959ebdbb 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -1,4 +1,3 @@ -#[allow(dead_code)] // mod control_plane; use control_plane::compute::ComputeControlPlane; use control_plane::storage::TestStorageControlPlane; diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 9333fdae29..b3f07f608c 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -13,7 +13,6 @@ pub mod waldecoder; pub mod walreceiver; pub mod walredo; -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct PageServerConf { pub data_dir: PathBuf, diff --git a/pageserver/src/tui_event.rs b/pageserver/src/tui_event.rs index c0e25da864..5546b680ee 100644 --- a/pageserver/src/tui_event.rs +++ b/pageserver/src/tui_event.rs @@ -10,7 +10,6 @@ use std::time::Duration; use termion::event::Key; use termion::input::TermRead; -#[allow(dead_code)] pub enum Event { Input(I), Tick, diff --git a/walkeeper/src/lib.rs b/walkeeper/src/lib.rs index 7dbb0b17f3..7e890cf98a 100644 --- a/walkeeper/src/lib.rs +++ b/walkeeper/src/lib.rs @@ -6,7 +6,6 @@ mod pq_protocol; pub mod wal_service; pub mod xlog_utils; -#[allow(dead_code)] #[derive(Debug, Clone)] pub struct WalAcceptorConf { pub data_dir: PathBuf, From e03417a7c9de041e4978e06da3f59ebbb4e5bc7e Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Sat, 17 Apr 2021 14:14:27 -0700 Subject: [PATCH 27/33] suppress dead_code warnings on nightly We don't need the nightly compiler, but there's no reason it shouldn't compile without warnings, either. I don't know why stable doesn't warn about these, but it's cheap to suppress them. --- pageserver/src/tui.rs | 1 + pageserver/src/waldecoder.rs | 2 ++ 2 files changed, 3 insertions(+) diff --git a/pageserver/src/tui.rs b/pageserver/src/tui.rs index 653600b82e..1cfd821cd9 100644 --- a/pageserver/src/tui.rs +++ b/pageserver/src/tui.rs @@ -188,6 +188,7 @@ pub fn ui_main<'b>() -> Result<(), Box> { Ok(()) } +#[allow(dead_code)] struct LogWidget<'a> { logger: &'a TuiLogger, title: &'a str, diff --git a/pageserver/src/waldecoder.rs b/pageserver/src/waldecoder.rs index 1f1a5dfc99..090256f318 100644 --- a/pageserver/src/waldecoder.rs +++ b/pageserver/src/waldecoder.rs @@ -44,6 +44,7 @@ struct XLogLongPageHeaderData { #[allow(non_upper_case_globals)] const SizeOfXLogLongPHD: usize = (2 + 2 + 4 + 8 + 4) + 4 + 8 + 4 + 4; +#[allow(dead_code)] pub struct WalStreamDecoder { lsn: u64, @@ -253,6 +254,7 @@ const BKPIMAGE_HAS_HOLE: u8 = 0x01; /* page image has "hole" */ const BKPIMAGE_IS_COMPRESSED: u8 = 0x02; /* page image is compressed */ const BKPIMAGE_APPLY: u8 = 0x04; /* page image should be restored during replay */ +#[allow(dead_code)] pub struct DecodedBkpBlock { /* Is this block ref in use? */ //in_use: bool, From 3c7f810849107fde3961f749cc935f1ca2f3622d Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Mon, 12 Apr 2021 13:59:32 -0700 Subject: [PATCH 28/33] clippy cleanup #1 Resolve some basic warnings from clippy: - useless conversion to the same type - redundant field names in struct initialization - redundant single-component path imports --- control_plane/src/compute.rs | 2 +- control_plane/src/local_env.rs | 1 - control_plane/src/storage.rs | 4 ++-- pageserver/src/bin/pageserver.rs | 3 --- pageserver/src/page_cache.rs | 12 ++++++------ pageserver/src/page_service.rs | 4 ++-- pageserver/src/restore_s3.rs | 6 +++--- pageserver/src/tui.rs | 1 - pageserver/src/tui_logger.rs | 3 +-- pageserver/src/waldecoder.rs | 6 +++--- pageserver/src/walreceiver.rs | 4 ++-- pageserver/src/walredo.rs | 2 +- walkeeper/src/bin/wal_acceptor.rs | 3 --- walkeeper/src/wal_service.rs | 8 ++++---- 14 files changed, 25 insertions(+), 34 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 30850dbd52..1d0db731f5 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -73,7 +73,7 @@ impl ComputeControlPlane { base_port: 65431, pageserver: Arc::clone(pageserver), nodes: BTreeMap::new(), - env: env.clone(), + env, } } diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 03cf982aa8..241fba2f62 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -9,7 +9,6 @@ use std::error; use std::fs; use std::path::{Path, PathBuf}; -use home; use serde_derive::{Deserialize, Serialize}; type Result = std::result::Result>; diff --git a/control_plane/src/storage.rs b/control_plane/src/storage.rs index 394f9b7aac..eba2966849 100644 --- a/control_plane/src/storage.rs +++ b/control_plane/src/storage.rs @@ -58,7 +58,7 @@ impl TestStorageControlPlane { let env = local_env::test_env(); let pserver = Arc::new(PageServerNode { - env: env.clone(), + env, kill_on_exit: true, listen_address: None, }); @@ -110,7 +110,7 @@ impl TestStorageControlPlane { pub fn get_wal_acceptor_conn_info(&self) -> String { self.wal_acceptors .iter() - .map(|wa| wa.listen.to_string().to_string()) + .map(|wa| wa.listen.to_string()) .collect::>() .join(",") } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 8bbe31fdd7..7419e3f894 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -14,10 +14,7 @@ use clap::{App, Arg}; use daemonize::Daemonize; use anyhow::Result; -use slog; use slog::Drain; -use slog_scope; -use slog_stdlog; use pageserver::page_service; use pageserver::restore_datadir; diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index a453c10a73..e820b218e3 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -205,7 +205,7 @@ pub struct CacheEntryContent { impl CacheEntry { fn new(key: CacheKey) -> CacheEntry { CacheEntry { - key: key, + key, content: Mutex::new(CacheEntryContent { page_image: None, wal_record: None, @@ -253,8 +253,8 @@ impl PageCache { // Look up cache entry. If it's a page image, return that. If it's a WAL record, // ask the WAL redo service to reconstruct the page image from the WAL records. - let minkey = CacheKey { tag: tag, lsn: 0 }; - let maxkey = CacheKey { tag: tag, lsn: lsn }; + let minkey = CacheKey { tag, lsn: 0 }; + let maxkey = CacheKey { tag, lsn }; let entry_rc: Arc; { @@ -302,7 +302,7 @@ impl PageCache { let entry_opt = entries.next_back(); if entry_opt.is_none() { - static ZERO_PAGE: [u8; 8192] = [0 as u8; 8192]; + static ZERO_PAGE: [u8; 8192] = [0u8; 8192]; return Ok(Bytes::from_static(&ZERO_PAGE)); /* return Err("could not find page image")?; */ } @@ -433,7 +433,7 @@ impl PageCache { // pub fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) { let key = CacheKey { - tag: tag, + tag, lsn: rec.lsn, }; @@ -469,7 +469,7 @@ impl PageCache { // Memorize a full image of a page version // pub fn put_page_image(&self, tag: BufferTag, lsn: u64, img: Bytes) { - let key = CacheKey { tag: tag, lsn: lsn }; + let key = CacheKey { tag, lsn }; let entry = CacheEntry::new(key.clone()); entry.content.lock().unwrap().page_image = Some(img); diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 1feb4d6f12..f704990f5e 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -259,7 +259,7 @@ impl Connection { stream: BufWriter::new(socket), buffer: BytesMut::with_capacity(10 * 1024), init_done: false, - conf: conf, + conf, } } @@ -560,7 +560,7 @@ impl Connection { self.write_message(&BeMessage::ZenithNblocksResponse(ZenithStatusResponse { ok: true, - n_blocks: n_blocks, + n_blocks, })) .await? } diff --git a/pageserver/src/restore_s3.rs b/pageserver/src/restore_s3.rs index 17e045fb5b..f3e642df67 100644 --- a/pageserver/src/restore_s3.rs +++ b/pageserver/src/restore_s3.rs @@ -60,8 +60,8 @@ pub fn restore_main(conf: &PageServerConf) { async fn restore_chunk(conf: &PageServerConf) -> Result<(), S3Error> { let backend = Storage { region: Region::Custom { - region: env::var("S3_REGION").unwrap().into(), - endpoint: env::var("S3_ENDPOINT").unwrap().into(), + region: env::var("S3_REGION").unwrap(), + endpoint: env::var("S3_ENDPOINT").unwrap(), }, credentials: Credentials::new( Some(&env::var("S3_ACCESSKEY").unwrap()), @@ -313,7 +313,7 @@ async fn slurp_base_file( dbnode: parsed.dbnode, relnode: parsed.relnode, forknum: parsed.forknum as u8, - blknum: blknum, + blknum, }; pcache.put_page_image(tag, parsed.lsn, bytes.copy_to_bytes(8192)); diff --git a/pageserver/src/tui.rs b/pageserver/src/tui.rs index 1cfd821cd9..4e35cc76c9 100644 --- a/pageserver/src/tui.rs +++ b/pageserver/src/tui.rs @@ -14,7 +14,6 @@ use tui::text::{Span, Spans, Text}; use tui::widgets::{Block, BorderType, Borders, Paragraph, Widget}; use tui::Terminal; -use slog; use slog::Drain; lazy_static! { diff --git a/pageserver/src/tui_logger.rs b/pageserver/src/tui_logger.rs index 0b49dcc388..e59ce15a56 100644 --- a/pageserver/src/tui_logger.rs +++ b/pageserver/src/tui_logger.rs @@ -10,7 +10,6 @@ // use chrono::offset::Local; use chrono::DateTime; -use slog; use slog::{Drain, Level, OwnedKVList, Record}; use slog_async::AsyncRecord; use std::collections::VecDeque; @@ -81,7 +80,7 @@ impl<'b> TuiLoggerWidget<'b> { style_trace: None, style_info: None, show_module: true, - logger: logger, + logger, } } } diff --git a/pageserver/src/waldecoder.rs b/pageserver/src/waldecoder.rs index 090256f318..957a103f4d 100644 --- a/pageserver/src/waldecoder.rs +++ b/pageserver/src/waldecoder.rs @@ -64,7 +64,7 @@ pub struct WalStreamDecoder { impl WalStreamDecoder { pub fn new(lsn: u64) -> WalStreamDecoder { WalStreamDecoder { - lsn: lsn, + lsn, startlsn: 0, contlen: 0, @@ -584,8 +584,8 @@ pub fn decode_wal_record(lsn: u64, rec: Bytes) -> DecodedWALRecord { // Since we don't care about the data payloads here, we're done. return DecodedWALRecord { - lsn: lsn, + lsn, record: rec, - blocks: blocks, + blocks, }; } diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index ecd5cd6ae8..56e27150e5 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -88,7 +88,7 @@ async fn walreceiver_main( // empty database snapshot), so for now I just put start of first segment which // seems to be a valid record. pcache.init_valid_lsn(0x_1_000_000_u64); - startpoint = u64::from(0x_1_000_000_u64); + startpoint = 0x_1_000_000_u64; } else { // There might be some padding after the last full record, skip it. // @@ -156,7 +156,7 @@ async fn walreceiver_main( }; let rec = page_cache::WALRecord { - lsn: lsn, + lsn, will_init: blk.will_init || blk.apply_image, rec: recdata.clone(), }; diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 9d5a1ee143..e3a0510080 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -208,7 +208,7 @@ impl WalRedoProcess { tokio::spawn(f_stderr); Ok(WalRedoProcess { - child: child, + child, stdin: RefCell::new(stdin), stdout: RefCell::new(stdout), }) diff --git a/walkeeper/src/bin/wal_acceptor.rs b/walkeeper/src/bin/wal_acceptor.rs index c8d635ba00..d50467ba49 100644 --- a/walkeeper/src/bin/wal_acceptor.rs +++ b/walkeeper/src/bin/wal_acceptor.rs @@ -11,10 +11,7 @@ use std::{fs::File, fs::OpenOptions}; use clap::{App, Arg}; -use slog; use slog::Drain; -use slog_scope; -use slog_stdlog; use walkeeper::wal_service; use walkeeper::WalAcceptorConf; diff --git a/walkeeper/src/wal_service.rs b/walkeeper/src/wal_service.rs index d70c10ce22..69632e3c28 100644 --- a/walkeeper/src/wal_service.rs +++ b/walkeeper/src/wal_service.rs @@ -402,7 +402,7 @@ impl System { }, }; System { - id: id, + id, mutex: Mutex::new(shared_state), cond: Notify::new(), } @@ -989,7 +989,7 @@ impl Connection { }, Err(ref e) if e.kind() == io::ErrorKind::WouldBlock => {} Err(e) => { - return Err(e.into()); + return Err(e); } } @@ -1018,7 +1018,7 @@ impl Connection { Ok(opened_file) => file = opened_file, Err(e) => { error!("Failed to open log file {:?}: {}", &wal_file_path, e); - return Err(e.into()); + return Err(e); } } } @@ -1130,7 +1130,7 @@ impl Connection { } Err(e) => { error!("Failed to open log file {:?}: {}", &wal_file_path, e); - return Err(e.into()); + return Err(e); } } } From b32cc6a088d2bf249c1fbc4c51f106801829cd6e Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Sun, 18 Apr 2021 19:15:53 -0700 Subject: [PATCH 29/33] pageserver: change over some error handling to anyhow+thiserror This is a first attempt at a new error-handling strategy: - Use anyhow::Error as the first choice for easy error handling - Use thiserror to generate local error types for anything that needs it (no error type is available to us) or will be inspected or matched on by higher layers. --- Cargo.lock | 1 + pageserver/Cargo.toml | 1 + pageserver/src/page_cache.rs | 38 ++++++++--------- pageserver/src/walreceiver.rs | 77 ++++++++++++++++------------------- 4 files changed, 53 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 74b32d20e3..85c299c7a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1215,6 +1215,7 @@ dependencies = [ "slog-stdlog", "slog-term", "termion", + "thiserror", "tokio", "tokio-postgres", "tokio-stream", diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 7f0b31e20c..f7f3be7f47 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -36,3 +36,4 @@ postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="a0d067b anyhow = "1.0" crc32c = "0.6.0" walkdir = "2" +thiserror = "1.0" diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index e820b218e3..1c05ea7e8f 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -6,25 +6,22 @@ // per-entry mutex. // +use crate::{walredo, PageServerConf}; +use anyhow::bail; +use bytes::Bytes; use core::ops::Bound::Included; +use crossbeam_channel::unbounded; +use crossbeam_channel::{Receiver, Sender}; +use lazy_static::lazy_static; +use log::*; +use rand::Rng; use std::collections::{BTreeMap, HashMap}; -use std::error::Error; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::sync::{Arc, Condvar, Mutex}; use std::thread; use std::time::Duration; use std::{convert::TryInto, ops::AddAssign}; -// use tokio::sync::RwLock; -use bytes::Bytes; -use lazy_static::lazy_static; -use log::*; -use rand::Rng; - -use crate::{walredo, PageServerConf}; - -use crossbeam_channel::unbounded; -use crossbeam_channel::{Receiver, Sender}; // Timeout when waiting or WAL receiver to catch up to an LSN given in a GetPage@LSN call. static TIMEOUT: Duration = Duration::from_secs(60); @@ -248,7 +245,7 @@ impl PageCache { // // Returns an 8k page image // - pub fn get_page_at_lsn(&self, tag: BufferTag, lsn: u64) -> Result> { + pub fn get_page_at_lsn(&self, tag: BufferTag, lsn: u64) -> anyhow::Result { self.num_getpage_requests.fetch_add(1, Ordering::Relaxed); // Look up cache entry. If it's a page image, return that. If it's a WAL record, @@ -276,11 +273,11 @@ impl PageCache { shared = wait_result.0; if wait_result.1.timed_out() { - return Err(format!( + bail!( "Timed out while waiting for WAL record at LSN {:X}/{:X} to arrive", lsn >> 32, lsn & 0xffff_ffff - ))?; + ); } } if waited { @@ -288,11 +285,11 @@ impl PageCache { } if lsn < shared.first_valid_lsn { - return Err(format!( + bail!( "LSN {:X}/{:X} has already been removed", lsn >> 32, lsn & 0xffff_ffff - ))?; + ); } let pagecache = &shared.pagecache; @@ -347,12 +344,12 @@ impl PageCache { error!( "could not apply WAL to reconstruct page image for GetPage@LSN request" ); - return Err("could not apply WAL to reconstruct page image".into()); + bail!("could not apply WAL to reconstruct page image"); } }; } else { // No base image, and no WAL record. Huh? - return Err(format!("no page image or WAL record for requested page"))?; + bail!("no page image or WAL record for requested page"); } } @@ -432,10 +429,7 @@ impl PageCache { // Adds a WAL record to the page cache // pub fn put_wal_record(&self, tag: BufferTag, rec: WALRecord) { - let key = CacheKey { - tag, - lsn: rec.lsn, - }; + let key = CacheKey { tag, lsn: rec.lsn }; let entry = CacheEntry::new(key.clone()); entry.content.lock().unwrap().wal_record = Some(rec); diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 56e27150e5..8c869c7346 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -1,28 +1,25 @@ -use std::str::FromStr; - -// -// WAL receiver -// -// The WAL receiver connects to the WAL safekeeper service, and streams WAL. -// For each WAL record, it decodes the record to figure out which data blocks -// the record affects, and adds the records to the page cache. -// -use log::*; - -use tokio::runtime; -use tokio::time::{sleep, Duration}; -use tokio_stream::StreamExt; +//! +//! WAL receiver +//! +//! The WAL receiver connects to the WAL safekeeper service, and streams WAL. +//! For each WAL record, it decodes the record to figure out which data blocks +//! the record affects, and adds the records to the page cache. +//! use crate::page_cache; use crate::page_cache::BufferTag; use crate::waldecoder::{decode_wal_record, WalStreamDecoder}; use crate::PageServerConf; - +use anyhow::Error; +use log::*; use postgres_protocol::message::backend::ReplicationMessage; use postgres_types::PgLsn; +use std::str::FromStr; +use tokio::runtime; +use tokio::time::{sleep, Duration}; use tokio_postgres::replication::{PgTimestamp, ReplicationStream}; - -use tokio_postgres::{Error, NoTls, SimpleQueryMessage, SimpleQueryRow}; +use tokio_postgres::{NoTls, SimpleQueryMessage, SimpleQueryRow}; +use tokio_stream::StreamExt; // // This is the entry point for the WAL receiver thread. @@ -49,10 +46,7 @@ pub fn thread_main(conf: &PageServerConf, wal_producer_connstr: &str) { }); } -async fn walreceiver_main( - conf: &PageServerConf, - wal_producer_connstr: &str, -) -> Result<(), Error> { +async fn walreceiver_main(conf: &PageServerConf, wal_producer_connstr: &str) -> Result<(), Error> { // Connect to the database in replication mode. info!("connecting to {:?}", wal_producer_connstr); let connect_cfg = format!("{} replication=true", wal_producer_connstr); @@ -109,10 +103,7 @@ async fn walreceiver_main( let startpoint = PgLsn::from(startpoint); let query = format!("START_REPLICATION PHYSICAL {}", startpoint); - let copy_stream = rclient - .copy_both_simple::(&query) - .await - .unwrap(); + let copy_stream = rclient.copy_both_simple::(&query).await?; let physical_stream = ReplicationStream::new(copy_stream); tokio::pin!(physical_stream); @@ -207,14 +198,13 @@ async fn walreceiver_main( let write_lsn = last_lsn; let flush_lsn = last_lsn; let apply_lsn = PgLsn::INVALID; - let ts = PgTimestamp::now().unwrap(); + let ts = PgTimestamp::now()?; const NO_REPLY: u8 = 0u8; physical_stream .as_mut() .standby_status_update(write_lsn, flush_lsn, apply_lsn, ts, NO_REPLY) - .await - .unwrap(); + .await?; } } _ => (), @@ -236,33 +226,36 @@ pub struct IdentifySystem { dbname: Option, } +/// There was a problem parsing the response to +/// a postgres IDENTIFY_SYSTEM command. +#[derive(Debug, thiserror::Error)] +#[error("IDENTIFY_SYSTEM parse error")] +pub struct IdentifyError; + /// Run the postgres `IDENTIFY_SYSTEM` command pub async fn identify_system(client: &tokio_postgres::Client) -> Result { let query_str = "IDENTIFY_SYSTEM"; let response = client.simple_query(query_str).await?; // get(N) from row, then parse it as some destination type. - fn get_parse(row: &SimpleQueryRow, idx: usize) -> Option + fn get_parse(row: &SimpleQueryRow, idx: usize) -> Result where T: FromStr, { - let val = row.get(idx)?; - val.parse::().ok() + let val = row.get(idx).ok_or(IdentifyError)?; + val.parse::().or(Err(IdentifyError)) } - // FIXME: turn unwrap() into errors. - // All of the tokio_postgres::Error builders are private, so we - // can't create them here. We'll just have to create our own error type. - - if let SimpleQueryMessage::Row(first_row) = response.get(0).unwrap() { + // extract the row contents into an IdentifySystem struct. + // written as a closure so I can use ? for Option here. + if let Some(SimpleQueryMessage::Row(first_row)) = response.get(0) { Ok(IdentifySystem { - systemid: get_parse(first_row, 0).unwrap(), - timeline: get_parse(first_row, 1).unwrap(), - xlogpos: get_parse(first_row, 2).unwrap(), - dbname: get_parse(first_row, 3), + systemid: get_parse(first_row, 0)?, + timeline: get_parse(first_row, 1)?, + xlogpos: get_parse(first_row, 2)?, + dbname: get_parse(first_row, 3).ok(), }) } else { - // FIXME: return an error - panic!("identify_system returned non-row response"); + Err(IdentifyError)? } } From 37258159354433564df273fe50bb3e16e2c8e875 Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Sun, 18 Apr 2021 22:34:01 -0700 Subject: [PATCH 30/33] pageserver: propage errors instead of calling .unwrap() Just a few more places where we can drop the .unwrap() call in favor of `?`. Also include a fix to the log file handling: don't open the file twice. Writing to two fds would result in one message overwriting another. Presumably `File.try_clone()` reduces down to `dup` on Linux. --- pageserver/src/bin/pageserver.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 7419e3f894..0ef258ad6c 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -8,11 +8,11 @@ use std::io; use std::path::PathBuf; use std::process::exit; use std::thread; -use std::{fs::File, fs::OpenOptions, str::FromStr}; +use std::{fs::File, fs::OpenOptions}; +use anyhow::{Context, Result}; use clap::{App, Arg}; use daemonize::Daemonize; -use anyhow::Result; use slog::Drain; @@ -88,11 +88,11 @@ fn main() -> Result<()> { } if let Some(addr) = arg_matches.value_of("wal_producer") { - conf.wal_producer_connstr = Some(String::from_str(addr).unwrap()); + conf.wal_producer_connstr = Some(String::from(addr)); } if let Some(addr) = arg_matches.value_of("listen") { - conf.listen_addr = addr.parse().unwrap(); + conf.listen_addr = addr.parse()?; } start_pageserver(&conf) @@ -101,7 +101,7 @@ fn main() -> Result<()> { fn start_pageserver(conf: &PageServerConf) -> Result<()> { // Initialize logger let _scope_guard = init_logging(&conf)?; - let _log_guard = slog_stdlog::init().unwrap(); + let _log_guard = slog_stdlog::init()?; // Note: this `info!(...)` macro comes from `log` crate info!("standard logging redirected to slog"); @@ -125,18 +125,15 @@ fn start_pageserver(conf: &PageServerConf) -> Result<()> { if conf.daemonize { info!("daemonizing..."); - // There should'n be any logging to stdin/stdout. Redirect it to the main log so - // that we will see any accidental manual fpritf's or backtraces. + // There shouldn't be any logging to stdin/stdout. Redirect it to the main log so + // that we will see any accidental manual fprintf's or backtraces. + let log_filename = conf.data_dir.join("pageserver.log"); let stdout = OpenOptions::new() .create(true) .append(true) - .open(conf.data_dir.join("pageserver.log")) - .unwrap(); - let stderr = OpenOptions::new() - .create(true) - .append(true) - .open(conf.data_dir.join("pageserver.log")) - .unwrap(); + .open(&log_filename) + .with_context(|| format!("failed to open {:?}", log_filename))?; + let stderr = stdout.try_clone()?; let daemonize = Daemonize::new() .pid_file(conf.data_dir.join("pageserver.pid")) From 8d1bf152cf371130385065252389b1641f54edaf Mon Sep 17 00:00:00 2001 From: Eric Seppanen Date: Sun, 18 Apr 2021 22:56:55 -0700 Subject: [PATCH 31/33] fix up logged error for walreceiver connection failed For some reason printing the Result made the error string print twice, along with some annoying newlines. Extracting the error first gets the expected result (just one explanation, no newlines) --- pageserver/src/walreceiver.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pageserver/src/walreceiver.rs b/pageserver/src/walreceiver.rs index 8c869c7346..5ca5ffa199 100644 --- a/pageserver/src/walreceiver.rs +++ b/pageserver/src/walreceiver.rs @@ -34,14 +34,15 @@ pub fn thread_main(conf: &PageServerConf, wal_producer_connstr: &str) { runtime.block_on(async { loop { - let _res = walreceiver_main(conf, wal_producer_connstr).await; + let res = walreceiver_main(conf, wal_producer_connstr).await; - // TODO: print/log the error - info!( - "WAL streaming connection failed, retrying in 1 second...: {:?}", - _res - ); - sleep(Duration::from_secs(1)).await; + if let Err(e) = res { + info!( + "WAL streaming connection failed ({}), retrying in 1 second", + e + ); + sleep(Duration::from_secs(1)).await; + } } }); } From 9809613c6f4cd3d334fc02a9d8d0ecf0110e9429 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 19 Apr 2021 10:34:51 +0300 Subject: [PATCH 32/33] Don't try to read from two WAL files in one read() call. That obviously won't work, you have to stop at the WAL file boundary, and open the next file. --- vendor/postgres | 2 +- walkeeper/src/wal_service.rs | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/vendor/postgres b/vendor/postgres index ea5a673983..167196910d 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit ea5a673983ac0d1e055b8da99d3dcca348cecc49 +Subproject commit 167196910d6f41466c82793bcf14bfe442468776 diff --git a/walkeeper/src/wal_service.rs b/walkeeper/src/wal_service.rs index 69632e3c28..5570781123 100644 --- a/walkeeper/src/wal_service.rs +++ b/walkeeper/src/wal_service.rs @@ -1023,7 +1023,14 @@ impl Connection { } } } - let send_size = min((end_pos - start_pos) as usize, MAX_SEND_SIZE); + let xlogoff = XLogSegmentOffset(start_pos, wal_seg_size) as usize; + + // How much to read and send in message? We cannot cross the WAL file + // boundary, and we don't want send more than MAX_SEND_SIZE. + let send_size = (end_pos - start_pos) as usize; + let send_size = min(send_size, wal_seg_size - xlogoff); + let send_size = min(send_size, MAX_SEND_SIZE); + let msg_size = LIBPQ_HDR_SIZE + XLOG_HDR_SIZE + send_size; let data_start = LIBPQ_HDR_SIZE + XLOG_HDR_SIZE; let data_end = data_start + send_size; From 8879f747eeb74288e2a561feb5792b3e0c9de2b7 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 19 Apr 2021 14:30:42 +0300 Subject: [PATCH 33/33] Add multitenancy test for wal_acceptor --- control_plane/src/compute.rs | 6 ++- integration_tests/tests/test_wal_acceptor.rs | 47 ++++++++++++++++++++ 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/control_plane/src/compute.rs b/control_plane/src/compute.rs index 1d0db731f5..b39d901be7 100644 --- a/control_plane/src/compute.rs +++ b/control_plane/src/compute.rs @@ -113,7 +113,7 @@ impl ComputeControlPlane { pub fn new_test_master_node(&mut self) -> Arc { let node = self.new_vanilla_node(true).unwrap(); - + println!("Create vanilla node at {:?}", node.address); node.append_conf( "postgresql.conf", "synchronous_standby_names = 'safekeeper_proxy'\n", @@ -405,7 +405,9 @@ impl PostgresNode { .args(&["-h", &self.address.ip().to_string()]) .args(&["-p", &self.address.port().to_string()]) .arg("-v") - .stderr(File::create(self.env.data_dir.join("safepkeeper_proxy.log")).unwrap()) + .stderr(OpenOptions::new() + .append(true) + .open(self.env.data_dir.join("safepkeeper_proxy.log")).unwrap()) .spawn() { Ok(child) => WalProposerNode { pid: child.id() }, diff --git a/integration_tests/tests/test_wal_acceptor.rs b/integration_tests/tests/test_wal_acceptor.rs index 523c568bb4..f4f7675b07 100644 --- a/integration_tests/tests/test_wal_acceptor.rs +++ b/integration_tests/tests/test_wal_acceptor.rs @@ -41,6 +41,53 @@ fn test_acceptors_normal_work() { // check wal files equality } +#[test] +fn test_multitenancy() { + // Start pageserver that reads WAL directly from that postgres + const REDUNDANCY: usize = 3; + const N_NODES: usize = 5; + let storage_cplane = TestStorageControlPlane::fault_tolerant(REDUNDANCY); + let mut compute_cplane = ComputeControlPlane::local(&storage_cplane.pageserver); + let wal_acceptors = storage_cplane.get_wal_acceptor_conn_info(); + + // start postgres + let mut nodes = Vec::new(); + let mut proxies = Vec::new(); + for _ in 0..N_NODES { + let node = compute_cplane.new_test_master_node(); + nodes.push(node); + nodes.last().unwrap().start().unwrap(); + proxies.push(nodes.last().unwrap().start_proxy(wal_acceptors.clone())); + } + + // create schema + for node in &nodes { + node.safe_psql( + "postgres", + "CREATE TABLE t(key int primary key, value text)", + ); + } + + // Populate data + for node in &nodes { + node.safe_psql( + "postgres", + "INSERT INTO t SELECT generate_series(1,100000), 'payload'", + ); + } + + // Check data + for node in &nodes { + let count: i64 = node + .safe_psql("postgres", "SELECT sum(key) FROM t") + .first() + .unwrap() + .get(0); + println!("sum = {}", count); + assert_eq!(count, 5000050000); + } +} + // Majority is always alive #[test] fn test_acceptors_restarts() {