From 65868258d2c8a46001c7b9426368941f87cec968 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 26 Jul 2024 11:03:44 +0100 Subject: [PATCH] tests: checkpoint instead of compact in test_sharding_split_compaction (#8473) ## Problem This test relies on writing image layers before the split. It can fail to do so durably if the image layers are written ahead of the remote consistent LSN, so we should have been doing a checkpoint rather than just a compaction --- test_runner/regress/test_sharding.py | 6 +++--- test_runner/regress/test_storage_scrubber.py | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index bc43bc77fa..f8770e70fe 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -198,8 +198,8 @@ def test_sharding_split_compaction(neon_env_builder: NeonEnvBuilder, failpoint: # disable background compaction and GC. We invoke it manually when we want it to happen. "gc_period": "0s", "compaction_period": "0s", - # create image layers eagerly, so that GC can remove some layers - "image_creation_threshold": 1, + # Disable automatic creation of image layers, as we will create them explicitly when we want them + "image_creation_threshold": 9999, "image_layer_creation_check_threshold": 0, } @@ -225,7 +225,7 @@ def test_sharding_split_compaction(neon_env_builder: NeonEnvBuilder, failpoint: # Do a full image layer generation before splitting, so that when we compact after splitting # we should only see sizes decrease (from post-split drops/rewrites), not increase (from image layer generation) - env.get_tenant_pageserver(tenant_id).http_client().timeline_compact( + env.get_tenant_pageserver(tenant_id).http_client().timeline_checkpoint( tenant_id, timeline_id, force_image_layer_creation=True, wait_until_uploaded=True ) diff --git a/test_runner/regress/test_storage_scrubber.py b/test_runner/regress/test_storage_scrubber.py index 7c411a6b84..a45430ca86 100644 --- a/test_runner/regress/test_storage_scrubber.py +++ b/test_runner/regress/test_storage_scrubber.py @@ -191,7 +191,9 @@ def test_scrubber_physical_gc_ancestors( "checkpoint_distance": f"{1024 * 1024}", "compaction_threshold": "1", "compaction_target_size": f"{1024 * 1024}", - "image_creation_threshold": "2", + # Disable automatic creation of image layers, as future image layers can result in layers in S3 that + # aren't referenced by children, earlier than the test expects such layers to exist + "image_creation_threshold": "9999", "image_layer_creation_check_threshold": "0", # Disable background compaction, we will do it explicitly "compaction_period": "0s", @@ -241,7 +243,7 @@ def test_scrubber_physical_gc_ancestors( workload.churn_rows(100) for shard in shards: ps = env.get_tenant_pageserver(shard) - ps.http_client().timeline_compact(shard, timeline_id) + ps.http_client().timeline_compact(shard, timeline_id, force_image_layer_creation=True) ps.http_client().timeline_gc(shard, timeline_id, 0) # We will use a min_age_secs=1 threshold for deletion, let it pass