From 7675d10588123002f3e3824f011c25e0619c4a6c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 9 Jan 2024 15:21:59 +0000 Subject: [PATCH] partial ruff format --- test_runner/fixtures/neon_fixtures.py | 46 +++++++++++++------ .../fixtures/pageserver/many_tenants.py | 8 +++- .../performance/test_pageserver_pagebench.py | 4 +- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 6aeac93f86..770d192fb7 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -25,7 +25,6 @@ from urllib.parse import urlparse import asyncpg import backoff import jwt -import psutil import psycopg2 import pytest import requests @@ -71,7 +70,6 @@ from fixtures.utils import ( subprocess_capture, wait_until, ) -from fixtures import overlayfs """ This file contains pytest fixtures. A fixture is a test resource that can be @@ -598,19 +596,33 @@ class NeonEnvBuilder: The overlayfs `upperdir` and `workdir` will be placed in test_overlay_dir. """ assert self.test_overlay_dir - assert self.test_output_dir in dstdir.parents # so that teardown & test_overlay_dir fixture work + assert ( + self.test_output_dir in dstdir.parents + ) # so that teardown & test_overlay_dir fixture work assert srcdir.is_dir() dstdir.mkdir(exist_ok=False, parents=False) ident_state_dir = self.test_overlay_dir / ident upper = ident_state_dir / "upper" work = ident_state_dir / "work" - ident_state_dir.mkdir(exist_ok=False, parents=False) # exists_ok=False also checks uniqueness in self.overlay_mounts + ident_state_dir.mkdir( + exist_ok=False, parents=False + ) # exists_ok=False also checks uniqueness in self.overlay_mounts upper.mkdir() work.mkdir() - cmd = [ "sudo", "mount", "-t", "overlay", "overlay", - "-o", f"lowerdir={srcdir},upperdir={upper},workdir={work}", str(dstdir) ] + cmd = [ + "sudo", + "mount", + "-t", + "overlay", + "overlay", + "-o", + f"lowerdir={srcdir},upperdir={upper},workdir={work}", + str(dstdir), + ] log.info(f"Mounting overlayfs srcdir={srcdir} dstdir={dstdir}: {cmd}") - subprocess_capture(self.test_output_dir, cmd, check=True, echo_stderr=True, echo_stdout=True) + subprocess_capture( + self.test_output_dir, cmd, check=True, echo_stderr=True, echo_stdout=True + ) self.overlay_mounts_created_by_us.append((ident, dstdir)) def overlay_cleanup_teardown(self): @@ -3314,9 +3326,11 @@ def get_test_overlay_dir(request: FixtureRequest, top_output_dir: Path) -> Path: """ return _get_test_dir(request, top_output_dir, "overlay-") + def get_test_snapshot_dir_path(request: FixtureRequest, top_output_dir: Path) -> Path: return _get_test_dir(request, top_output_dir, "snapshot-") + def get_test_repo_dir(request: FixtureRequest, top_output_dir: Path) -> Path: return get_test_output_dir(request, top_output_dir) / "repo" @@ -3352,7 +3366,7 @@ def test_output_dir( ) -> Iterator[Path]: """Create the working directory for an individual test.""" - _ = test_overlay_dir # request it so it can do cleanups + _ = test_overlay_dir # request it so it can do cleanups # one directory per test test_dir = get_test_output_dir(request, top_output_dir) @@ -3364,6 +3378,7 @@ def test_output_dir( allure_attach_from_dir(test_dir) + class SnapshotDir: _path: Path @@ -3392,10 +3407,12 @@ class SnapshotDir: @pytest.fixture(scope="function", autouse=True) -def test_snapshot_dir(request: FixtureRequest, top_output_dir: Path, test_overlay_dir: Path) -> SnapshotDir: +def test_snapshot_dir( + request: FixtureRequest, top_output_dir: Path, test_overlay_dir: Path +) -> SnapshotDir: """Create the working directory for an individual test.""" - _ = test_overlay_dir # overlay mounts use snapshot dir as the lowerdir => request it so it can unmount stale overlay mounts first + _ = test_overlay_dir # overlay mounts use snapshot dir as the lowerdir => request it so it can unmount stale overlay mounts first # one directory per test snapshot_dir = get_test_snapshot_dir_path(request, top_output_dir) @@ -3403,7 +3420,6 @@ def test_snapshot_dir(request: FixtureRequest, top_output_dir: Path, test_overla return SnapshotDir(snapshot_dir) - @pytest.fixture(scope="function", autouse=True) def test_overlay_dir(request: FixtureRequest, top_output_dir: Path) -> Optional[Path]: """Create the overlay state directory for an individual test.""" @@ -3418,11 +3434,13 @@ def test_overlay_dir(request: FixtureRequest, top_output_dir: Path) -> Optional[ overlay_dir.mkdir(exist_ok=True) # unmount stale overlayfs mounts which subdirectories of `overlay_dir/*` as the overlayfs `upperdir` and `workdir` for mountpoint in overlayfs.iter_mounts_beneath(get_test_output_dir(request, top_output_dir)): - cmd = [ "sudo", "umount", str(mountpoint) ] - log.info(f"Unmounting stale overlayfs mount probably created during earlier test run: {cmd}") + cmd = ["sudo", "umount", str(mountpoint)] + log.info( + f"Unmounting stale overlayfs mount probably created during earlier test run: {cmd}" + ) subprocess.run(cmd, capture_output=True, check=True) # the overlayfs `workdir`` is owned by `root` - cmd = [ "sudo", "rm", "-rf", str(overlay_dir)] + cmd = ["sudo", "rm", "-rf", str(overlay_dir)] subprocess.run(cmd, capture_output=True, check=True) overlay_dir.mkdir() diff --git a/test_runner/fixtures/pageserver/many_tenants.py b/test_runner/fixtures/pageserver/many_tenants.py index d3baf69d31..f3b9d563bb 100644 --- a/test_runner/fixtures/pageserver/many_tenants.py +++ b/test_runner/fixtures/pageserver/many_tenants.py @@ -61,7 +61,9 @@ def single_timeline( log.info("invoking callback to create template tenant") template_tenant, template_timeline, template_config = setup_template(env) - log.info(f"template tenant is template_tenant={template_tenant} template_timeline={template_timeline}") + log.info( + f"template tenant is template_tenant={template_tenant} template_timeline={template_timeline}" + ) log.info(f"detach template tenant form pageserver") env.pageserver.http_client().tenant_detach(template_tenant) @@ -90,7 +92,9 @@ def single_timeline( work_queue.do(22, tenants, attach_broken) - env.pageserver.stop(immediate=True) # clears the failpoint as a side-effect; immediate to avoid hitting neon_local's timeout + env.pageserver.stop( + immediate=True + ) # clears the failpoint as a side-effect; immediate to avoid hitting neon_local's timeout tenant_timelines = list(map(lambda tenant: (tenant, template_timeline), tenants)) log.info(f"python-side on-demand download the layer files into local tenant dir") fixtures.pageserver.remote_storage.copy_all_remote_layer_files_to_local_tenant_dir( diff --git a/test_runner/performance/test_pageserver_pagebench.py b/test_runner/performance/test_pageserver_pagebench.py index 8a0d36bc90..55409090ff 100644 --- a/test_runner/performance/test_pageserver_pagebench.py +++ b/test_runner/performance/test_pageserver_pagebench.py @@ -92,6 +92,8 @@ def test_getpage_throughput( log.info(f"Results:\n{json.dumps(results, sort_keys=True, indent=2)}") - env.pageserver.stop(immediate=True) # with 20k tenants, we hit neon_local's shutdown timeout of 10 seconds + env.pageserver.stop( + immediate=True + ) # with 20k tenants, we hit neon_local's shutdown timeout of 10 seconds zenbenchmark.record_pagebench_results("get-page-latest-lsn", results, duration)