From 46acdaed8d9d3b6509a80efbbe6e60b2958b1eee Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 12 May 2023 12:06:38 +0200 Subject: [PATCH] follow-up: fix all the test cases --- pageserver/src/pgdatadir_mapping.rs | 10 +- pageserver/src/tenant.rs | 116 ++++++++++++------ .../src/tenant/remote_timeline_client.rs | 10 +- .../walreceiver/connection_manager.rs | 3 +- pageserver/src/walingest.rs | 8 +- 5 files changed, 98 insertions(+), 49 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 67f37ee519..33d60de2de 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -1594,15 +1594,17 @@ fn is_slru_block_key(key: Key) -> bool { } #[cfg(test)] -pub fn create_test_timeline( +pub async fn create_test_timeline( tenant: &crate::tenant::Tenant, timeline_id: utils::id::TimelineId, pg_version: u32, ctx: &RequestContext, ) -> anyhow::Result> { let tline = tenant - .create_empty_timeline(timeline_id, Lsn(8), pg_version, ctx)? - .initialize(ctx)?; + .create_empty_timeline(timeline_id, Lsn(8), pg_version, ctx) + .await? + .initialize(ctx) + .await?; let mut m = tline.begin_modification(Lsn(8)); m.init_empty()?; m.commit()?; @@ -1632,7 +1634,7 @@ mod tests { #[test] fn test_list_rels_drop() -> Result<()> { let repo = RepoHarness::create("test_list_rels_drop")?.load(); - let tline = create_empty_timeline(repo, TIMELINE_ID)?; + let tline = create_empty_timeline(repo, TIMELINE_ID).await?; const TESTDB: u32 = 111; // Import initial dummy checkpoint record, otherwise the get_timeline() call diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 07206cfc7f..862d339d82 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3210,8 +3210,10 @@ mod tests { #[tokio::test] async fn test_basic() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_basic")?.load().await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; let writer = tline.writer(); writer.put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10")))?; @@ -3244,11 +3246,15 @@ mod tests { let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines")? .load() .await; - let timeline = - tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let _ = timeline.initialize(&ctx)?; + let timeline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let _ = timeline.initialize(&ctx).await?; - match tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) { + match tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await + { Ok(_) => panic!("duplicate timeline creation should fail"), Err(e) => assert_eq!( e.to_string(), @@ -3277,8 +3283,10 @@ mod tests { use std::str::from_utf8; let (tenant, ctx) = TenantHarness::create("test_branch")?.load().await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; let writer = tline.writer(); #[allow(non_snake_case)] @@ -3304,6 +3312,7 @@ mod tests { .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) + .await .expect("Should have a local timeline"); let new_writer = newtline.writer(); new_writer.put(TEST_KEY_A, Lsn(0x40), &test_value("bar at 0x40"))?; @@ -3375,8 +3384,10 @@ mod tests { TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? .load() .await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50 @@ -3414,8 +3425,10 @@ mod tests { .await; let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx)? - .initialize(&ctx)?; + .create_empty_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx) + .await? + .initialize(&ctx) + .await?; // try to branch at lsn 0x25, should fail because initdb lsn is 0x50 match tenant .branch_timeline(&tline, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) @@ -3444,7 +3457,7 @@ mod tests { RepoHarness::create("test_prohibit_get_for_garbage_collected_data")? .load(); - let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION)?; + let tline = repo.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION).await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; repo.gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO)?; @@ -3465,8 +3478,10 @@ mod tests { .load() .await; let tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)? - .initialize(&ctx)?; + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await? + .initialize(&ctx) + .await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant @@ -3474,6 +3489,7 @@ mod tests { .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) + .await .expect("Should have a local timeline"); make_some_layers(newtline.as_ref(), Lsn(0x60)).await?; @@ -3502,8 +3518,10 @@ mod tests { TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? .load() .await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant @@ -3511,6 +3529,7 @@ mod tests { .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) + .await .expect("Should have a local timeline"); // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50 tenant @@ -3526,8 +3545,10 @@ mod tests { TenantHarness::create("test_parent_keeps_data_forever_after_branching")? .load() .await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant @@ -3535,6 +3556,7 @@ mod tests { .await?; let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) + .await .expect("Should have a local timeline"); make_some_layers(newtline.as_ref(), Lsn(0x60)).await?; @@ -3559,15 +3581,17 @@ mod tests { let harness = TenantHarness::create(TEST_NAME)?; { let (tenant, ctx) = harness.load().await; - let tline = - tenant.create_empty_timeline(TIMELINE_ID, Lsn(0x8000), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0x8000), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; make_some_layers(tline.as_ref(), Lsn(0x8000)).await?; } let (tenant, _ctx) = harness.load().await; tenant .get_timeline(TIMELINE_ID, true) + .await .expect("cannot load timeline"); Ok(()) @@ -3580,9 +3604,10 @@ mod tests { // create two timelines { let (tenant, ctx) = harness.load().await; - let tline = - tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; @@ -3592,6 +3617,7 @@ mod tests { let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) + .await .expect("Should have a local timeline"); make_some_layers(newtline.as_ref(), Lsn(0x60)).await?; @@ -3603,10 +3629,12 @@ mod tests { // check that both, child and ancestor are loaded let _child_tline = tenant .get_timeline(NEW_TIMELINE_ID, true) + .await .expect("cannot get child timeline loaded"); let _ancestor_tline = tenant .get_timeline(TIMELINE_ID, true) + .await .expect("cannot get ancestor timeline loaded"); Ok(()) @@ -3619,8 +3647,10 @@ mod tests { let (tenant, ctx) = harness.load().await; tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)? - .initialize(&ctx)?; + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await? + .initialize(&ctx) + .await?; drop(tenant); let metadata_path = harness.timeline_path(&TIMELINE_ID).join(METADATA_FILE_NAME); @@ -3657,8 +3687,10 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_images")?.load().await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; let writer = tline.writer(); writer.put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10")))?; @@ -3723,8 +3755,10 @@ mod tests { #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_bulk_insert")?.load().await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; let mut lsn = Lsn(0x10); @@ -3766,8 +3800,10 @@ mod tests { #[tokio::test] async fn test_random_updates() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_random_updates")?.load().await; - let tline = tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let tline = tline.initialize(&ctx)?; + let tline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let tline = tline.initialize(&ctx).await?; const NUM_KEYS: usize = 1000; @@ -3841,8 +3877,10 @@ mod tests { .load() .await; let mut tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)? - .initialize(&ctx)?; + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await? + .initialize(&ctx) + .await?; const NUM_KEYS: usize = 1000; @@ -3879,6 +3917,7 @@ mod tests { .await?; tline = tenant .get_timeline(new_tline_id, true) + .await .expect("Should have the branched timeline"); for _ in 0..NUM_KEYS { @@ -3925,8 +3964,10 @@ mod tests { .load() .await; let mut tline = tenant - .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)? - .initialize(&ctx)?; + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await? + .initialize(&ctx) + .await?; const NUM_KEYS: usize = 100; const NUM_TLINES: usize = 50; @@ -3945,6 +3986,7 @@ mod tests { .await?; tline = tenant .get_timeline(new_tline_id, true) + .await .expect("Should have the branched timeline"); for _ in 0..NUM_KEYS { diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 96aabd7945..11a194ee42 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1264,9 +1264,13 @@ mod tests { let harness = TenantHarness::create(test_name)?; let (tenant, ctx) = runtime.block_on(harness.load()); // create an empty timeline directory - let timeline = - tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let _ = timeline.initialize(&ctx).unwrap(); + runtime.block_on(async { + let timeline = tenant + .create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) + .await?; + let _ = timeline.initialize(&ctx).await?; + anyhow::Ok(()) + })?; let remote_fs_dir = harness.conf.workdir.join("remote_fs"); std::fs::create_dir_all(remote_fs_dir)?; diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index 9cb17ea799..746d3c597e 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -1334,8 +1334,9 @@ mod tests { let (tenant, ctx) = harness.load().await; let timeline = tenant .create_empty_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION, &ctx) + .await .expect("Failed to create an empty timeline for dummy wal connection manager"); - let timeline = timeline.initialize(&ctx).unwrap(); + let timeline = timeline.initialize(&ctx).await.unwrap(); ConnectionManagerState { id: TenantTimelineId { diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 63d568a342..7c9953ce86 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1200,7 +1200,7 @@ mod tests { #[tokio::test] async fn test_relsize() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_relsize")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx).await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut m = tline.begin_modification(Lsn(0x20)); @@ -1419,7 +1419,7 @@ mod tests { #[tokio::test] async fn test_drop_extend() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_drop_extend")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx).await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut m = tline.begin_modification(Lsn(0x20)); @@ -1488,7 +1488,7 @@ mod tests { #[tokio::test] async fn test_truncate_extend() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_truncate_extend")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx).await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; // Create a 20 MB relation (the size is arbitrary) @@ -1628,7 +1628,7 @@ mod tests { #[tokio::test] async fn test_large_rel() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_large_rel")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx).await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut lsn = 0x10;