From fbff37a64c25a4b4fa36f4766abde98f91ac74bb Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sun, 1 Aug 2021 13:44:02 +0300 Subject: [PATCH] Merge fixes --- pageserver/src/branches.rs | 31 ++++++++++++------- pageserver/src/layered_repository.rs | 4 ++- pageserver/src/page_cache.rs | 3 +- pageserver/src/repository.rs | 2 +- test_runner/batch_others/test_snapfiles_gc.py | 12 +++---- 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/pageserver/src/branches.rs b/pageserver/src/branches.rs index 0d860837ab..c012e1b96b 100644 --- a/pageserver/src/branches.rs +++ b/pageserver/src/branches.rs @@ -25,7 +25,7 @@ use crate::page_cache; use crate::restore_local_repo; use crate::walredo::WalRedoManager; use crate::ZTenantId; -use crate::{repository::Repository, PageServerConf, ZTimelineId}; +use crate::{repository::Repository, PageServerConf, RepositoryFormat, ZTimelineId}; #[derive(Serialize, Deserialize, Clone)] pub struct BranchInfo { @@ -73,8 +73,8 @@ pub fn init_pageserver( pub fn create_repo( conf: &'static PageServerConf, tenantid: ZTenantId, - wal_redo_manager: Arc, -) -> Result { + wal_redo_manager: Arc, +) -> Result> { let repo_dir = conf.tenant_path(&tenantid); if repo_dir.exists() { bail!("repo for {} already exists", tenantid) @@ -104,19 +104,28 @@ pub fn create_repo( // and we failed to run initdb again in the same directory. This has been solved for the // rapid init+start case now, but the general race condition remains if you restart the // server quickly. - let storage = crate::rocksdb_storage::RocksObjectStore::create(conf, &tenantid)?; + let repo: Arc = match conf.repository_format { + RepositoryFormat::Layered => Arc::new( + crate::layered_repository::LayeredRepository::new(conf, + wal_redo_manager, + tenantid + )), + RepositoryFormat::RocksDb => { + let obj_store = crate::rocksdb_storage::RocksObjectStore::open(conf, &tenantid).unwrap(); - let repo = crate::object_repository::ObjectRepository::new( - conf, - std::sync::Arc::new(storage), - wal_redo_manager, - tenantid, - ); + Arc::new(ObjectRepository::new( + conf, + Arc::new(obj_store), + wal_redo_manager, + tenantid + )) + } + }; // Load data into pageserver // TODO To implement zenith import we need to // move data loading out of create_repo() - bootstrap_timeline(conf, tenantid, tli, &repo)?; + bootstrap_timeline(conf, tenantid, tli, &*repo)?; Ok(repo) } diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 86c0b2804b..e66fe47240 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -201,6 +201,8 @@ impl LayeredRepository { let path = conf.timeline_path(&timelineid, &tenantid).join("metadata"); let mut file = File::create(&path)?; + info!("saving metadata {}", path.display()); + file.write_all(&TimelineMetadata::ser(data)?)?; Ok(()) @@ -256,7 +258,7 @@ impl LayeredRepository { // Remember timelineid and its last_record_lsn for each timeline let mut timelines: Vec<(ZTimelineId, Lsn)> = Vec::new(); - let timelines_path = conf.workdir.join("timelines"); + let timelines_path = conf.timelines_path(&tenantid); for direntry in fs::read_dir(timelines_path)? { let direntry = direntry?; if let Some(fname) = direntry.file_name().to_str() { diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index ecc756a4b0..a9d1ffe635 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -24,7 +24,6 @@ lazy_static! { pub fn init(conf: &'static PageServerConf) { let mut m = REPOSITORY.lock().unwrap(); - for dir_entry in fs::read_dir(conf.tenants_path()).unwrap() { let tenantid = ZTenantId::from_str(dir_entry.unwrap().file_name().to_str().unwrap()).unwrap(); @@ -68,7 +67,7 @@ pub fn create_repository_for_tenant( let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenantid)); let repo = branches::create_repo(conf, tenantid, wal_redo_manager)?; - m.insert(tenantid, Arc::new(repo)); + m.insert(tenantid, repo); Ok(()) } diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 855ab79da0..adfcc981f0 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -362,7 +362,7 @@ mod tests { let repo: Box = match conf.repository_format { RepositoryFormat::Layered => { - Box::new(LayeredRepository::new(conf, Arc::new(walredo_mgr))) + Box::new(LayeredRepository::new(conf, Arc::new(walredo_mgr), tenantid)) } RepositoryFormat::RocksDb => { let obj_store = RocksObjectStore::create(conf, &tenantid)?; diff --git a/test_runner/batch_others/test_snapfiles_gc.py b/test_runner/batch_others/test_snapfiles_gc.py index d7a0984690..1cd05d1aa8 100644 --- a/test_runner/batch_others/test_snapfiles_gc.py +++ b/test_runner/batch_others/test_snapfiles_gc.py @@ -42,7 +42,7 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin): cur.execute("VACUUM") print("Running GC before test") - pscur.execute(f"do_gc {timeline} 0") + pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row); # remember the number of files @@ -54,7 +54,7 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin): snapshot_relfiles_remain += 1; print("Inserting one row and running GC") cur.execute("INSERT INTO foo VALUES (1)") - pscur.execute(f"do_gc {timeline} 0") + pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row); assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain @@ -68,7 +68,7 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin): cur.execute("INSERT INTO foo VALUES (2)") cur.execute("INSERT INTO foo VALUES (3)") - pscur.execute(f"do_gc {timeline} 0") + pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row); assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1 @@ -80,7 +80,7 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin): cur.execute("INSERT INTO foo VALUES (2)") cur.execute("INSERT INTO foo VALUES (3)") - pscur.execute(f"do_gc {timeline} 0") + pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row); assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain + 1 @@ -89,7 +89,7 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin): # Run GC again, with no changes in the database. Should not remove anything. print("Run GC again, with nothing to do") - pscur.execute(f"do_gc {timeline} 0") + pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row); assert row['snapshot_relfiles_total'] == snapshot_relfiles_remain @@ -102,7 +102,7 @@ def test_snapfiles_gc(zenith_cli, pageserver, postgres, pg_bin): print("Drop table and run GC again"); cur.execute("DROP TABLE foo") - pscur.execute(f"do_gc {timeline} 0") + pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") row = pscur.fetchone() print_gc_result(row);