From c660926a062632e12bb46cb2effd824dc004b809 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 8 Oct 2021 00:34:26 +0300 Subject: [PATCH] Refactor duplicated code to get on-disk timeline size in tests. Move it to a common function. In the passing, remove the obsolete check to exclude the 'wal' directory. The 'wal' directory is no more. --- test_runner/fixtures/benchmark_fixture.py | 13 +++++++++++++ test_runner/performance/test_bulk_insert.py | 15 +-------------- test_runner/performance/test_perf_pgbench.py | 15 +-------------- 3 files changed, 15 insertions(+), 28 deletions(-) diff --git a/test_runner/fixtures/benchmark_fixture.py b/test_runner/fixtures/benchmark_fixture.py index e972b7bf9e..a2a8cc64d3 100644 --- a/test_runner/fixtures/benchmark_fixture.py +++ b/test_runner/fixtures/benchmark_fixture.py @@ -151,6 +151,19 @@ class ZenithBenchmarker: re.MULTILINE) return int(round(float(matches.group(1)))) + def get_timeline_size(repo_dir: str, tenantid: str, timelineid: str): + """ + Calculate the on-disk size of a timeline + """ + path = "{}/tenants/{}/timelines/{}".format(repo_dir, tenantid, timelineid) + + totalbytes = 0 + for root, dirs, files in os.walk(path): + for name in files: + totalbytes += os.path.getsize(os.path.join(root, name)) + + return totalbytes + @contextmanager def record_pageserver_writes(self, pageserver, metric_name): """ diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py index 65a5ca8de9..95f1ea5e4a 100644 --- a/test_runner/performance/test_bulk_insert.py +++ b/test_runner/performance/test_bulk_insert.py @@ -4,19 +4,6 @@ from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") -def get_timeline_size(repo_dir: str, tenantid: str, timelineid: str): - path = "{}/tenants/{}/timelines/{}".format(repo_dir, tenantid, timelineid) - - totalbytes = 0 - for root, dirs, files in os.walk(path): - for name in files: - totalbytes += os.path.getsize(os.path.join(root, name)) - - if 'wal' in dirs: - dirs.remove('wal') # don't visit 'wal' subdirectory - - return totalbytes - # # Run bulk INSERT test. # @@ -60,5 +47,5 @@ def test_bulk_insert(postgres: PostgresFactory, pageserver: ZenithPageserver, pg zenbenchmark.record("peak_mem", zenbenchmark.get_peak_mem(pageserver) / 1024, 'MB') # Report disk space used by the repository - timeline_size = get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) + timeline_size = zenbenchmark.get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) zenbenchmark.record('size', timeline_size / (1024*1024), 'MB') diff --git a/test_runner/performance/test_perf_pgbench.py b/test_runner/performance/test_perf_pgbench.py index 7e0f19bec8..18db78f12a 100644 --- a/test_runner/performance/test_perf_pgbench.py +++ b/test_runner/performance/test_perf_pgbench.py @@ -4,19 +4,6 @@ from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture") -def get_timeline_size(repo_dir: str, tenantid: str, timelineid: str): - path = "{}/tenants/{}/timelines/{}".format(repo_dir, tenantid, timelineid) - - totalbytes = 0 - for root, dirs, files in os.walk(path): - for name in files: - totalbytes += os.path.getsize(os.path.join(root, name)) - - if 'wal' in dirs: - dirs.remove('wal') # don't visit 'wal' subdirectory - - return totalbytes - # # Run a very short pgbench test. # @@ -64,5 +51,5 @@ def test_pgbench(postgres: PostgresFactory, pageserver: ZenithPageserver, pg_bin pscur.execute(f"do_gc {pageserver.initial_tenant} {timeline} 0") # Report disk space used by the repository - timeline_size = get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) + timeline_size = zenbenchmark.get_timeline_size(repo_dir, pageserver.initial_tenant, timeline) zenbenchmark.record('size', timeline_size / (1024*1024), 'MB')