From b9902004965c717810d19302bc9f613a480a66bf Mon Sep 17 00:00:00 2001 From: Dmitry Rodionov Date: Fri, 30 Jun 2023 15:01:06 +0300 Subject: [PATCH] tests: use shortcut everywhere to get timeline path (#4586) --- test_runner/regress/test_disk_usage_eviction.py | 15 +++++++-------- test_runner/regress/test_layer_eviction.py | 2 +- test_runner/regress/test_remote_storage.py | 2 +- test_runner/regress/test_tenant_detach.py | 12 ++++++------ test_runner/regress/test_tenant_relocation.py | 4 +--- .../regress/test_tenants_with_remote_storage.py | 2 +- 6 files changed, 17 insertions(+), 20 deletions(-) diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index 0ec023b9e1..fc2d29c2c1 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -1,7 +1,6 @@ import shutil import time from dataclasses import dataclass -from pathlib import Path from typing import Dict, Tuple import pytest @@ -428,14 +427,14 @@ def poor_mans_du( largest_layer = 0 smallest_layer = None for tenant_id, timeline_id in timelines: - dir = Path(env.repo_dir) / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) - assert dir.exists(), f"timeline dir does not exist: {dir}" - sum = 0 - for file in dir.iterdir(): + timeline_dir = env.timeline_dir(tenant_id, timeline_id) + assert timeline_dir.exists(), f"timeline dir does not exist: {timeline_dir}" + total = 0 + for file in timeline_dir.iterdir(): if "__" not in file.name: continue size = file.stat().st_size - sum += size + total += size largest_layer = max(largest_layer, size) if smallest_layer: smallest_layer = min(smallest_layer, size) @@ -443,8 +442,8 @@ def poor_mans_du( smallest_layer = size log.info(f"{tenant_id}/{timeline_id} => {file.name} {size}") - log.info(f"{tenant_id}/{timeline_id}: sum {sum}") - total_on_disk += sum + log.info(f"{tenant_id}/{timeline_id}: sum {total}") + total_on_disk += total assert smallest_layer is not None or total_on_disk == 0 and largest_layer == 0 return (total_on_disk, largest_layer, smallest_layer or 0) diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index b22e545f20..d4f0c94a29 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -58,7 +58,7 @@ def test_basic_eviction( for sk in env.safekeepers: sk.stop() - timeline_path = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_path = env.timeline_dir(tenant_id, timeline_id) initial_local_layers = sorted( list(filter(lambda path: path.name != "metadata", timeline_path.glob("*"))) ) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index f2b954a822..9a70266314 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -535,7 +535,7 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( "pitr_interval": "0s", } ) - timeline_path = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_path = env.timeline_dir(tenant_id, timeline_id) client = env.pageserver.http_client() diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 2a015d5d17..2ded79954e 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -632,14 +632,14 @@ def test_ignored_tenant_download_missing_layers( # ignore the tenant and remove its layers pageserver_http.tenant_ignore(tenant_id) - tenant_timeline_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_dir = env.timeline_dir(tenant_id, timeline_id) layers_removed = False - for dir_entry in tenant_timeline_dir.iterdir(): + for dir_entry in timeline_dir.iterdir(): if dir_entry.name.startswith("00000"): # Looks like a layer file. Remove it dir_entry.unlink() layers_removed = True - assert layers_removed, f"Found no layers for tenant {tenant_timeline_dir}" + assert layers_removed, f"Found no layers for tenant {timeline_dir}" # now, load it from the local files and expect it to work due to remote storage restoration pageserver_http.tenant_load(tenant_id=tenant_id) @@ -688,14 +688,14 @@ def test_ignored_tenant_stays_broken_without_metadata( # ignore the tenant and remove its metadata pageserver_http.tenant_ignore(tenant_id) - tenant_timeline_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_dir = env.timeline_dir(tenant_id, timeline_id) metadata_removed = False - for dir_entry in tenant_timeline_dir.iterdir(): + for dir_entry in timeline_dir.iterdir(): if dir_entry.name == "metadata": # Looks like a layer file. Remove it dir_entry.unlink() metadata_removed = True - assert metadata_removed, f"Failed to find metadata file in {tenant_timeline_dir}" + assert metadata_removed, f"Failed to find metadata file in {timeline_dir}" env.pageserver.allowed_errors.append( f".*{tenant_id}.*: load failed.*: failed to load metadata.*" diff --git a/test_runner/regress/test_tenant_relocation.py b/test_runner/regress/test_tenant_relocation.py index 70b931d7f9..9043c29060 100644 --- a/test_runner/regress/test_tenant_relocation.py +++ b/test_runner/regress/test_tenant_relocation.py @@ -214,9 +214,7 @@ def switch_pg_to_new_pageserver( endpoint.start() - timeline_to_detach_local_path = ( - env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) - ) + timeline_to_detach_local_path = env.timeline_dir(tenant_id, timeline_id) files_before_detach = os.listdir(timeline_to_detach_local_path) assert ( "metadata" in files_before_detach diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index dca2cd3d28..98f9e94276 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -257,7 +257,7 @@ def test_tenant_redownloads_truncated_file_on_startup( env.endpoints.stop_all() env.pageserver.stop() - timeline_dir = Path(env.repo_dir) / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_dir = env.timeline_dir(tenant_id, timeline_id) local_layer_truncated = None for path in Path.iterdir(timeline_dir): if path.name.startswith("00000"):