From 9a4fbf365ce69e36c974abe1242fe01bcc63dd56 Mon Sep 17 00:00:00 2001 From: anastasia Date: Fri, 2 Apr 2021 17:45:11 +0300 Subject: [PATCH] add test test_pageserver_recovery. zenith_push postgres to minio and start pageserver using this base image --- integration_tests/tests/control_plane/mod.rs | 56 ++++++- integration_tests/tests/test_pageserver.rs | 150 +++++++++++++------ vendor/postgres | 2 +- 3 files changed, 159 insertions(+), 49 deletions(-) diff --git a/integration_tests/tests/control_plane/mod.rs b/integration_tests/tests/control_plane/mod.rs index eab3f345af..9e7099cddb 100644 --- a/integration_tests/tests/control_plane/mod.rs +++ b/integration_tests/tests/control_plane/mod.rs @@ -57,7 +57,7 @@ pub struct StorageControlPlane { impl StorageControlPlane { // postgres <-> page_server - pub fn one_page_server() -> StorageControlPlane { + pub fn one_page_server(froms3: bool) -> StorageControlPlane { let mut cplane = StorageControlPlane { wal_acceptors: Vec::new(), page_servers: Vec::new(), @@ -68,7 +68,14 @@ impl StorageControlPlane { data_dir: TEST_WORKDIR.join("pageserver"), }; pserver.init(); - pserver.start(); + if froms3 + { + pserver.start_froms3(); + } + else + { + pserver.start(); + } cplane.page_servers.push(pserver); cplane @@ -186,6 +193,28 @@ impl PageServerNode { } } + pub fn start_froms3(&self) { + println!("Starting pageserver at '{}'", self.page_service_addr); + + let status = Command::new(CARGO_BIN_DIR.join("pageserver")) + .args(&["-D", self.data_dir.to_str().unwrap()]) + .args(&["-l", self.page_service_addr.to_string().as_str()]) + .arg("-d") + .env_clear() + .env("PATH", PG_BIN_DIR.to_str().unwrap()) // path to postres-wal-redo binary + .env("S3_ENDPOINT", "https://192.168.16.216:9000") + .env("S3_REGION", "us-east-1") + .env("S3_ACCESSKEY", "minioadmin") + .env("S3_SECRET", "minioadmin") + .env("S3_BUCKET", "zenith-testbucket") + .status() + .expect("failed to start pageserver"); + + if !status.success() { + panic!("pageserver start failed"); + } + } + pub fn stop(&self) { let pidfile = self.data_dir.join("pageserver.pid"); let pid = fs::read_to_string(pidfile).unwrap(); @@ -613,6 +642,29 @@ impl PostgresNode { } } + pub fn push_to_s3(&self) + { + println!("Push to s3 node at '{}'", self.pgdata.to_str().unwrap()); + + let zenith_push_path = self.pg_bin_dir.join("zenith_push"); + println!("zenith_push_path: {}", zenith_push_path.to_str().unwrap()); + + let status = Command::new(zenith_push_path) + .args(&["-D", self.pgdata.to_str().unwrap()]) + .env_clear() + .env("S3_ENDPOINT", "https://192.168.16.216:9000") + .env("S3_REGION", "us-east-1") + .env("S3_ACCESSKEY", "minioadmin") + .env("S3_SECRET", "minioadmin") + // .env("S3_BUCKET", "zenith-testbucket") + .status() + .expect("failed to push node to s3"); + + if !status.success() { + panic!("zenith_push failed"); + } + } + // TODO pub fn pg_bench() {} pub fn pg_regress() {} diff --git a/integration_tests/tests/test_pageserver.rs b/integration_tests/tests/test_pageserver.rs index 8adacb3c54..8342d3821d 100644 --- a/integration_tests/tests/test_pageserver.rs +++ b/integration_tests/tests/test_pageserver.rs @@ -9,55 +9,106 @@ use control_plane::StorageControlPlane; // -- pageserver api endpoint to check all rels // Handcrafted cases with wal records that are (were) problematic for redo. +// #[test] +// fn test_redo_cases() { +// // Start pageserver that reads WAL directly from that postgres +// let storage_cplane = StorageControlPlane::one_page_server(false); +// let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + +// // start postgres +// let node = compute_cplane.new_node(); +// node.start(&storage_cplane); + +// println!("await pageserver connection..."); +// sleep(Duration::from_secs(3)); + +// // check basic work with table +// node.safe_psql("postgres", "CREATE TABLE t(key int primary key, value text)"); +// node.safe_psql("postgres", "INSERT INTO t SELECT generate_series(1,100000), 'payload'"); +// let count: i64 = node +// .safe_psql("postgres", "SELECT sum(key) FROM t") +// .first() +// .unwrap() +// .get(0); +// println!("sum = {}", count); +// assert_eq!(count, 5000050000); + +// // check 'create table as' +// node.safe_psql("postgres", "CREATE TABLE t2 AS SELECT * FROM t"); +// let count: i64 = node +// .safe_psql("postgres", "SELECT sum(key) FROM t") +// .first() +// .unwrap() +// .get(0); +// println!("sum = {}", count); +// assert_eq!(count, 5000050000); +// } + +// // Runs pg_regress on a compute node +// #[test] +// fn test_regress() { +// // Start pageserver that reads WAL directly from that postgres +// let storage_cplane = StorageControlPlane::one_page_server(false); +// let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + +// // start postgres +// let node = compute_cplane.new_node(); +// node.start(&storage_cplane); + +// println!("await pageserver connection..."); +// sleep(Duration::from_secs(3)); + +// pageserver::control_plane::regress_check(&node); +// } + +// // Run two postgres instances on one pageserver +// #[test] +// fn test_pageserver_multitenancy() { +// // Start pageserver that reads WAL directly from that postgres +// let storage_cplane = StorageControlPlane::one_page_server(false); +// let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); + +// // Allocate postgres instance, but don't start +// let node1 = compute_cplane.new_node(); +// let node2 = compute_cplane.new_node(); +// node1.start(&storage_cplane); +// node2.start(&storage_cplane); + +// // XXX: add some extension func to postgres to check walsender conn +// // XXX: or better just drop that +// println!("await pageserver connection..."); +// sleep(Duration::from_secs(3)); + +// // check node1 +// node1.safe_psql("postgres", "CREATE TABLE t(key int primary key, value text)"); +// node1.safe_psql("postgres", "INSERT INTO t SELECT generate_series(1,100000), 'payload'"); +// let count: i64 = node1 +// .safe_psql("postgres", "SELECT sum(key) FROM t") +// .first() +// .unwrap() +// .get(0); +// println!("sum = {}", count); +// assert_eq!(count, 5000050000); + +// // check node2 +// node2.safe_psql("postgres", "CREATE TABLE t(key int primary key, value text)"); +// node2.safe_psql("postgres", "INSERT INTO t SELECT generate_series(100000,200000), 'payload'"); +// let count: i64 = node2 +// .safe_psql("postgres", "SELECT sum(key) FROM t") +// .first() +// .unwrap() +// .get(0); +// println!("sum = {}", count); +// assert_eq!(count, 15000150000); +// } + #[test] -fn test_redo_cases() { - // Start pageserver that reads WAL directly from that postgres - let storage_cplane = StorageControlPlane::one_page_server(); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); +// Start pageserver using s3 base image +fn test_pageserver_recovery() { - // start postgres - let node = compute_cplane.new_node(); - node.start(&storage_cplane); - - // check basic work with table - node.safe_psql( - "postgres", - "CREATE TABLE t(key int primary key, value text)", - ); - node.safe_psql( - "postgres", - "INSERT INTO t SELECT generate_series(1,100000), 'payload'", - ); - let count: i64 = node - .safe_psql("postgres", "SELECT sum(key) FROM t") - .first() - .unwrap() - .get(0); - println!("sum = {}", count); - assert_eq!(count, 5000050000); - - // check 'create table as' - node.safe_psql("postgres", "CREATE TABLE t2 AS SELECT * FROM t"); - let count: i64 = node - .safe_psql("postgres", "SELECT sum(key) FROM t") - .first() - .unwrap() - .get(0); - println!("sum = {}", count); - assert_eq!(count, 5000050000); -} - -// Runs pg_regress on a compute node -#[test] -fn test_regress() { - // Start pageserver that reads WAL directly from that postgres - let storage_cplane = StorageControlPlane::one_page_server(); - let mut compute_cplane = ComputeControlPlane::local(&storage_cplane); - - // start postgres - let node = compute_cplane.new_node(); - node.start(&storage_cplane); + let storage_cplane = StorageControlPlane::one_page_server(true); +<<<<<<< HEAD:integration_tests/tests/test_pageserver.rs control_plane::regress_check(&node); } @@ -107,4 +158,11 @@ fn test_pageserver_multitenancy() { .get(0); println!("sum = {}", count); assert_eq!(count, 15000150000); +======= + sleep(Duration::from_secs(30)); + + + //Request page from pageserver + storage_cplane.page_server_psql("controlfile"); +>>>>>>> add test test_pageserver_recovery.:tests/test_pageserver.rs } diff --git a/vendor/postgres b/vendor/postgres index b1f5a5ec14..254c3cc109 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit b1f5a5ec145d5d9614eec4824074edae1162e5fa +Subproject commit 254c3cc1099d78cfcc431da9cc46c28ae944b59c