From 876efcfc0ad1d7ecce726ae8cc167a3dad99aa2b Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Sun, 27 Aug 2023 16:53:14 +0000 Subject: [PATCH] REPRO the problem: , uses 430GB of space; 4 seconds load time; constant 20kIOPS after ~20s --- .../src/tenant/storage_layer/delta_layer.rs | 10 +++--- .../src/tenant/storage_layer/image_layer.rs | 10 +++--- .../test_pageserver_startup_many_tenants.py | 32 +++++++++++++++++++ 3 files changed, 42 insertions(+), 10 deletions(-) create mode 100644 test_runner/performance/test_pageserver_startup_many_tenants.py diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index a235034f76..707745cce5 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -857,11 +857,11 @@ impl DeltaLayerInner { expected_summary.index_start_blk = actual_summary.index_start_blk; expected_summary.index_root_blk = actual_summary.index_root_blk; if actual_summary != expected_summary { - bail!( - "in-file summary does not match expected summary. actual = {:?} expected = {:?}", - actual_summary, - expected_summary - ); + // bail!( + // "in-file summary does not match expected summary. actual = {:?} expected = {:?}", + // actual_summary, + // expected_summary + // ); } } diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 2d0d3a3dad..6f43bc1fa7 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -450,11 +450,11 @@ impl ImageLayerInner { expected_summary.index_root_blk = actual_summary.index_root_blk; if actual_summary != expected_summary { - bail!( - "in-file summary does not match expected summary. actual = {:?} expected = {:?}", - actual_summary, - expected_summary - ); + // bail!( + // "in-file summary does not match expected summary. actual = {:?} expected = {:?}", + // actual_summary, + // expected_summary + // ); } } diff --git a/test_runner/performance/test_pageserver_startup_many_tenants.py b/test_runner/performance/test_pageserver_startup_many_tenants.py new file mode 100644 index 0000000000..3bf0244a7c --- /dev/null +++ b/test_runner/performance/test_pageserver_startup_many_tenants.py @@ -0,0 +1,32 @@ +import queue +import threading +from fixtures.neon_fixtures import NeonEnvBuilder, PgBin, wait_for_last_flush_lsn +from fixtures.types import TenantId + + +def test_pageserver_startup_many_tenants(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): + env = neon_env_builder.init_start() + + # below doesn't work because summaries contain tenant and timeline ids and we check for them + + tenant_id, timeline_id = env.initial_tenant, env.initial_timeline + pshttp = env.pageserver.http_client() + ep = env.endpoints.create_start("main") + ep.safe_psql("create table foo(b text)") + for i in range(0, 8): + ep.safe_psql("insert into foo(b) values ('some text')") + # pg_bin.run_capture(["pgbench", "-i", "-s1", ep.connstr()]) + wait_for_last_flush_lsn(env, ep, tenant_id, timeline_id) + pshttp.timeline_checkpoint(tenant_id, timeline_id) + ep.stop_and_destroy() + + env.pageserver.stop() + for sk in env.safekeepers: + sk.stop() + + tenant_dir = env.repo_dir / "tenants" / str(env.initial_tenant) + + for i in range(0, 20_000): + import shutil + + shutil.copytree(tenant_dir, tenant_dir.parent / str(TenantId.generate()))