diff --git a/test_runner/fixtures/broker.py b/test_runner/fixtures/broker.py new file mode 100644 index 0000000000..fa8b816e69 --- /dev/null +++ b/test_runner/fixtures/broker.py @@ -0,0 +1,60 @@ +import subprocess +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Optional + +from fixtures.log_helper import log + + +@dataclass +class NeonBroker: + """An object managing storage_broker instance""" + + logfile: Path + port: int + neon_binpath: Path + handle: Optional[subprocess.Popen[Any]] = None # handle of running daemon + + def listen_addr(self): + return f"127.0.0.1:{self.port}" + + def client_url(self): + return f"http://{self.listen_addr()}" + + def check_status(self): + return True # TODO + + def try_start(self): + if self.handle is not None: + log.debug(f"storage_broker is already running on port {self.port}") + return + + listen_addr = self.listen_addr() + log.info(f'starting storage_broker to listen incoming connections at "{listen_addr}"') + with open(self.logfile, "wb") as logfile: + args = [ + str(self.neon_binpath / "storage_broker"), + f"--listen-addr={listen_addr}", + ] + self.handle = subprocess.Popen(args, stdout=logfile, stderr=logfile) + + # wait for start + started_at = time.time() + while True: + try: + self.check_status() + except Exception as e: + elapsed = time.time() - started_at + if elapsed > 5: + raise RuntimeError( + f"timed out waiting {elapsed:.0f}s for storage_broker start: {e}" + ) from e + time.sleep(0.5) + else: + break # success + + def stop(self): + if self.handle is not None: + self.handle.terminate() + self.handle.wait() diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 392c4d4c0b..ac237cafca 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2,13 +2,11 @@ from __future__ import annotations import abc import asyncio -import enum import filecmp import json import os import re import shutil -import socket import subprocess import tempfile import textwrap @@ -17,12 +15,11 @@ import uuid from contextlib import closing, contextmanager from dataclasses import dataclass, field from datetime import datetime -from enum import Flag, auto from functools import cached_property from itertools import chain, product from pathlib import Path from types import TracebackType -from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, Union, cast +from typing import Any, Dict, Iterator, List, Optional, Tuple, Type, cast from urllib.parse import urlparse import asyncpg @@ -42,10 +39,21 @@ from psycopg2.extensions import cursor as PgCursor from psycopg2.extensions import make_dsn, parse_dsn from typing_extensions import Literal +from fixtures.broker import NeonBroker from fixtures.log_helper import log from fixtures.pageserver.http import PageserverHttpClient from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload from fixtures.pg_version import PgVersion +from fixtures.port_distributor import PortDistributor +from fixtures.remote_storage import ( + LocalFsStorage, + MockS3Server, + RemoteStorage, + RemoteStorageKind, + RemoteStorageUsers, + S3Storage, + remote_storage_to_toml_inline_table, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import ( ATTACHMENT_NAME_REGEX, @@ -220,77 +228,6 @@ def get_dir_size(path: str) -> int: return totalbytes -def can_bind(host: str, port: int) -> bool: - """ - Check whether a host:port is available to bind for listening - - Inspired by the can_bind() perl function used in Postgres tests, in - vendor/postgres-v14/src/test/perl/PostgresNode.pm - """ - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: - # TODO: The pageserver and safekeepers don't use SO_REUSEADDR at the - # moment. If that changes, we should use start using SO_REUSEADDR here - # too, to allow reusing ports more quickly. - # See https://github.com/neondatabase/neon/issues/801 - # sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - try: - sock.bind((host, port)) - sock.listen() - return True - except socket.error: - log.info(f"Port {port} is in use, skipping") - return False - finally: - sock.close() - - -class PortDistributor: - def __init__(self, base_port: int, port_number: int): - self.iterator = iter(range(base_port, base_port + port_number)) - self.port_map: Dict[int, int] = {} - - def get_port(self) -> int: - for port in self.iterator: - if can_bind("localhost", port): - return port - raise RuntimeError( - "port range configured for test is exhausted, consider enlarging the range" - ) - - def replace_with_new_port(self, value: Union[int, str]) -> Union[int, str]: - """ - Returns a new port for a port number in a string (like "localhost:1234") or int. - Replacements are memorised, so a substitution for the same port is always the same. - """ - - # TODO: replace with structural pattern matching for Python >= 3.10 - if isinstance(value, int): - return self._replace_port_int(value) - - if isinstance(value, str): - return self._replace_port_str(value) - - raise TypeError(f"unsupported type {type(value)} of {value=}") - - def _replace_port_int(self, value: int) -> int: - known_port = self.port_map.get(value) - if known_port is None: - known_port = self.port_map[value] = self.get_port() - - return known_port - - def _replace_port_str(self, value: str) -> str: - # Use regex to find port in a string - # urllib.parse.urlparse produces inconvenient results for cases without scheme like "localhost:5432" - # See https://bugs.python.org/issue27657 - ports = re.findall(r":(\d+)(?:/|$)", value) - assert len(ports) == 1, f"can't find port in {value}" - port_int = int(ports[0]) - - return value.replace(f":{port_int}", f":{self._replace_port_int(port_int)}") - - @pytest.fixture(scope="session") def port_distributor(worker_base_port: int) -> PortDistributor: return PortDistributor(base_port=worker_base_port, port_number=WORKER_PORT_NUM) @@ -464,140 +401,6 @@ class AuthKeys: return self.generate_token(scope="tenant", tenant_id=str(tenant_id)) -class MockS3Server: - """ - Starts a mock S3 server for testing on a port given, errors if the server fails to start or exits prematurely. - Relies that `poetry` and `moto` server are installed, since it's the way the tests are run. - - Also provides a set of methods to derive the connection properties from and the method to kill the underlying server. - """ - - def __init__( - self, - port: int, - ): - self.port = port - - # XXX: do not use `shell=True` or add `exec ` to the command here otherwise. - # We use `self.subprocess.kill()` to shut down the server, which would not "just" work in Linux - # if a process is started from the shell process. - self.subprocess = subprocess.Popen(["poetry", "run", "moto_server", "s3", f"-p{port}"]) - error = None - try: - return_code = self.subprocess.poll() - if return_code is not None: - error = f"expected mock s3 server to run but it exited with code {return_code}. stdout: '{self.subprocess.stdout}', stderr: '{self.subprocess.stderr}'" - except Exception as e: - error = f"expected mock s3 server to start but it failed with exception: {e}. stdout: '{self.subprocess.stdout}', stderr: '{self.subprocess.stderr}'" - if error is not None: - log.error(error) - self.kill() - raise RuntimeError("failed to start s3 mock server") - - def endpoint(self) -> str: - return f"http://127.0.0.1:{self.port}" - - def region(self) -> str: - return "us-east-1" - - def access_key(self) -> str: - return "test" - - def secret_key(self) -> str: - return "test" - - def kill(self): - self.subprocess.kill() - - -@enum.unique -class RemoteStorageKind(str, enum.Enum): - LOCAL_FS = "local_fs" - MOCK_S3 = "mock_s3" - REAL_S3 = "real_s3" - # Pass to tests that are generic to remote storage - # to ensure the test pass with or without the remote storage - NOOP = "noop" - - -def available_remote_storages() -> List[RemoteStorageKind]: - remote_storages = [RemoteStorageKind.LOCAL_FS, RemoteStorageKind.MOCK_S3] - if os.getenv("ENABLE_REAL_S3_REMOTE_STORAGE") is not None: - remote_storages.append(RemoteStorageKind.REAL_S3) - log.info("Enabling real s3 storage for tests") - else: - log.info("Using mock implementations to test remote storage") - return remote_storages - - -def available_s3_storages() -> List[RemoteStorageKind]: - remote_storages = [RemoteStorageKind.MOCK_S3] - if os.getenv("ENABLE_REAL_S3_REMOTE_STORAGE") is not None: - remote_storages.append(RemoteStorageKind.REAL_S3) - log.info("Enabling real s3 storage for tests") - else: - log.info("Using mock implementations to test remote storage") - return remote_storages - - -@dataclass -class LocalFsStorage: - root: Path - - -@dataclass -class S3Storage: - bucket_name: str - bucket_region: str - access_key: str - secret_key: str - endpoint: Optional[str] = None - prefix_in_bucket: Optional[str] = "" - - def access_env_vars(self) -> Dict[str, str]: - return { - "AWS_ACCESS_KEY_ID": self.access_key, - "AWS_SECRET_ACCESS_KEY": self.secret_key, - } - - def to_string(self) -> str: - return json.dumps( - { - "bucket": self.bucket_name, - "region": self.bucket_region, - "endpoint": self.endpoint, - "prefix": self.prefix_in_bucket, - } - ) - - -RemoteStorage = Union[LocalFsStorage, S3Storage] - - -# serialize as toml inline table -def remote_storage_to_toml_inline_table(remote_storage: RemoteStorage) -> str: - if isinstance(remote_storage, LocalFsStorage): - remote_storage_config = f"local_path='{remote_storage.root}'" - elif isinstance(remote_storage, S3Storage): - remote_storage_config = f"bucket_name='{remote_storage.bucket_name}',\ - bucket_region='{remote_storage.bucket_region}'" - - if remote_storage.prefix_in_bucket is not None: - remote_storage_config += f",prefix_in_bucket='{remote_storage.prefix_in_bucket}'" - - if remote_storage.endpoint is not None: - remote_storage_config += f",endpoint='{remote_storage.endpoint}'" - else: - raise Exception("invalid remote storage type") - - return f"{{{remote_storage_config}}}" - - -class RemoteStorageUsers(Flag): - PAGESERVER = auto() - SAFEKEEPER = auto() - - class NeonEnvBuilder: """ Builder object to create a Neon runtime environment @@ -2899,59 +2702,6 @@ class SafekeeperHttpClient(requests.Session): return metrics -@dataclass -class NeonBroker: - """An object managing storage_broker instance""" - - logfile: Path - port: int - neon_binpath: Path - handle: Optional[subprocess.Popen[Any]] = None # handle of running daemon - - def listen_addr(self): - return f"127.0.0.1:{self.port}" - - def client_url(self): - return f"http://{self.listen_addr()}" - - def check_status(self): - return True # TODO - - def try_start(self): - if self.handle is not None: - log.debug(f"storage_broker is already running on port {self.port}") - return - - listen_addr = self.listen_addr() - log.info(f'starting storage_broker to listen incoming connections at "{listen_addr}"') - with open(self.logfile, "wb") as logfile: - args = [ - str(self.neon_binpath / "storage_broker"), - f"--listen-addr={listen_addr}", - ] - self.handle = subprocess.Popen(args, stdout=logfile, stderr=logfile) - - # wait for start - started_at = time.time() - while True: - try: - self.check_status() - except Exception as e: - elapsed = time.time() - started_at - if elapsed > 5: - raise RuntimeError( - f"timed out waiting {elapsed:.0f}s for storage_broker start: {e}" - ) from e - time.sleep(0.5) - else: - break # success - - def stop(self): - if self.handle is not None: - self.handle.terminate() - self.handle.wait() - - def get_test_output_dir(request: FixtureRequest, top_output_dir: Path) -> Path: """Compute the working directory for an individual test.""" test_name = request.node.name diff --git a/test_runner/fixtures/port_distributor.py b/test_runner/fixtures/port_distributor.py new file mode 100644 index 0000000000..fd808d7a5f --- /dev/null +++ b/test_runner/fixtures/port_distributor.py @@ -0,0 +1,77 @@ +import re +import socket +from contextlib import closing +from typing import Dict, Union + +from fixtures.log_helper import log + + +def can_bind(host: str, port: int) -> bool: + """ + Check whether a host:port is available to bind for listening + + Inspired by the can_bind() perl function used in Postgres tests, in + vendor/postgres-v14/src/test/perl/PostgresNode.pm + """ + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock: + # TODO: The pageserver and safekeepers don't use SO_REUSEADDR at the + # moment. If that changes, we should use start using SO_REUSEADDR here + # too, to allow reusing ports more quickly. + # See https://github.com/neondatabase/neon/issues/801 + # sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + try: + sock.bind((host, port)) + sock.listen() + return True + except socket.error: + log.info(f"Port {port} is in use, skipping") + return False + finally: + sock.close() + + +class PortDistributor: + def __init__(self, base_port: int, port_number: int): + self.iterator = iter(range(base_port, base_port + port_number)) + self.port_map: Dict[int, int] = {} + + def get_port(self) -> int: + for port in self.iterator: + if can_bind("localhost", port): + return port + raise RuntimeError( + "port range configured for test is exhausted, consider enlarging the range" + ) + + def replace_with_new_port(self, value: Union[int, str]) -> Union[int, str]: + """ + Returns a new port for a port number in a string (like "localhost:1234") or int. + Replacements are memorised, so a substitution for the same port is always the same. + """ + + # TODO: replace with structural pattern matching for Python >= 3.10 + if isinstance(value, int): + return self._replace_port_int(value) + + if isinstance(value, str): + return self._replace_port_str(value) + + raise TypeError(f"unsupported type {type(value)} of {value=}") + + def _replace_port_int(self, value: int) -> int: + known_port = self.port_map.get(value) + if known_port is None: + known_port = self.port_map[value] = self.get_port() + + return known_port + + def _replace_port_str(self, value: str) -> str: + # Use regex to find port in a string + # urllib.parse.urlparse produces inconvenient results for cases without scheme like "localhost:5432" + # See https://bugs.python.org/issue27657 + ports = re.findall(r":(\d+)(?:/|$)", value) + assert len(ports) == 1, f"can't find port in {value}" + port_int = int(ports[0]) + + return value.replace(f":{port_int}", f":{self._replace_port_int(port_int)}") diff --git a/test_runner/fixtures/remote_storage.py b/test_runner/fixtures/remote_storage.py new file mode 100644 index 0000000000..1b80473377 --- /dev/null +++ b/test_runner/fixtures/remote_storage.py @@ -0,0 +1,143 @@ +import enum +import json +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Optional, Union + +from fixtures.log_helper import log + + +class MockS3Server: + """ + Starts a mock S3 server for testing on a port given, errors if the server fails to start or exits prematurely. + Relies that `poetry` and `moto` server are installed, since it's the way the tests are run. + + Also provides a set of methods to derive the connection properties from and the method to kill the underlying server. + """ + + def __init__( + self, + port: int, + ): + self.port = port + + # XXX: do not use `shell=True` or add `exec ` to the command here otherwise. + # We use `self.subprocess.kill()` to shut down the server, which would not "just" work in Linux + # if a process is started from the shell process. + self.subprocess = subprocess.Popen(["poetry", "run", "moto_server", "s3", f"-p{port}"]) + error = None + try: + return_code = self.subprocess.poll() + if return_code is not None: + error = f"expected mock s3 server to run but it exited with code {return_code}. stdout: '{self.subprocess.stdout}', stderr: '{self.subprocess.stderr}'" + except Exception as e: + error = f"expected mock s3 server to start but it failed with exception: {e}. stdout: '{self.subprocess.stdout}', stderr: '{self.subprocess.stderr}'" + if error is not None: + log.error(error) + self.kill() + raise RuntimeError("failed to start s3 mock server") + + def endpoint(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def region(self) -> str: + return "us-east-1" + + def access_key(self) -> str: + return "test" + + def secret_key(self) -> str: + return "test" + + def kill(self): + self.subprocess.kill() + + +@enum.unique +class RemoteStorageKind(str, enum.Enum): + LOCAL_FS = "local_fs" + MOCK_S3 = "mock_s3" + REAL_S3 = "real_s3" + # Pass to tests that are generic to remote storage + # to ensure the test pass with or without the remote storage + NOOP = "noop" + + +def available_remote_storages() -> List[RemoteStorageKind]: + remote_storages = [RemoteStorageKind.LOCAL_FS, RemoteStorageKind.MOCK_S3] + if os.getenv("ENABLE_REAL_S3_REMOTE_STORAGE") is not None: + remote_storages.append(RemoteStorageKind.REAL_S3) + log.info("Enabling real s3 storage for tests") + else: + log.info("Using mock implementations to test remote storage") + return remote_storages + + +def available_s3_storages() -> List[RemoteStorageKind]: + remote_storages = [RemoteStorageKind.MOCK_S3] + if os.getenv("ENABLE_REAL_S3_REMOTE_STORAGE") is not None: + remote_storages.append(RemoteStorageKind.REAL_S3) + log.info("Enabling real s3 storage for tests") + else: + log.info("Using mock implementations to test remote storage") + return remote_storages + + +@dataclass +class LocalFsStorage: + root: Path + + +@dataclass +class S3Storage: + bucket_name: str + bucket_region: str + access_key: str + secret_key: str + endpoint: Optional[str] = None + prefix_in_bucket: Optional[str] = "" + + def access_env_vars(self) -> Dict[str, str]: + return { + "AWS_ACCESS_KEY_ID": self.access_key, + "AWS_SECRET_ACCESS_KEY": self.secret_key, + } + + def to_string(self) -> str: + return json.dumps( + { + "bucket": self.bucket_name, + "region": self.bucket_region, + "endpoint": self.endpoint, + "prefix": self.prefix_in_bucket, + } + ) + + +RemoteStorage = Union[LocalFsStorage, S3Storage] + + +# serialize as toml inline table +def remote_storage_to_toml_inline_table(remote_storage: RemoteStorage) -> str: + if isinstance(remote_storage, LocalFsStorage): + remote_storage_config = f"local_path='{remote_storage.root}'" + elif isinstance(remote_storage, S3Storage): + remote_storage_config = f"bucket_name='{remote_storage.bucket_name}',\ + bucket_region='{remote_storage.bucket_region}'" + + if remote_storage.prefix_in_bucket is not None: + remote_storage_config += f",prefix_in_bucket='{remote_storage.prefix_in_bucket}'" + + if remote_storage.endpoint is not None: + remote_storage_config += f",endpoint='{remote_storage.endpoint}'" + else: + raise Exception("invalid remote storage type") + + return f"{{{remote_storage_config}}}" + + +class RemoteStorageUsers(enum.Flag): + PAGESERVER = enum.auto() + SAFEKEEPER = enum.auto() diff --git a/test_runner/regress/test_attach_tenant_config.py b/test_runner/regress/test_attach_tenant_config.py index b6b46f9401..bc6afa84a1 100644 --- a/test_runner/regress/test_attach_tenant_config.py +++ b/test_runner/regress/test_attach_tenant_config.py @@ -3,12 +3,11 @@ from typing import Generator, Optional import pytest from fixtures.neon_fixtures import ( - LocalFsStorage, NeonEnv, NeonEnvBuilder, - RemoteStorageKind, ) from fixtures.pageserver.http import PageserverApiException, TenantConfig +from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.types import TenantId from fixtures.utils import wait_until diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index eb26270a5f..fa386ee75a 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -13,7 +13,6 @@ from fixtures.neon_fixtures import ( NeonCli, NeonEnvBuilder, PgBin, - PortDistributor, ) from fixtures.pageserver.http import PageserverHttpClient from fixtures.pageserver.utils import ( @@ -22,6 +21,7 @@ from fixtures.pageserver.utils import ( wait_for_upload, ) from fixtures.pg_version import PgVersion +from fixtures.port_distributor import PortDistributor from fixtures.types import Lsn from pytest import FixtureRequest diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index fc2d29c2c1..1e9b130b1c 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -7,15 +7,14 @@ import pytest import toml from fixtures.log_helper import log from fixtures.neon_fixtures import ( - LocalFsStorage, NeonEnv, NeonEnvBuilder, PgBin, - RemoteStorageKind, wait_for_last_flush_lsn, ) from fixtures.pageserver.http import PageserverHttpClient from fixtures.pageserver.utils import wait_for_upload_queue_empty +from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import wait_until diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py index b7692b71c6..c4bee1a1ea 100644 --- a/test_runner/regress/test_download_extensions.py +++ b/test_runner/regress/test_download_extensions.py @@ -8,10 +8,9 @@ import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, - RemoteStorageKind, - available_s3_storages, ) from fixtures.pg_version import PgVersion +from fixtures.remote_storage import RemoteStorageKind, available_s3_storages # Cleaning up downloaded files is important for local tests diff --git a/test_runner/regress/test_fullbackup.py b/test_runner/regress/test_fullbackup.py index ece9dccf93..214f1f33a8 100644 --- a/test_runner/regress/test_fullbackup.py +++ b/test_runner/regress/test_fullbackup.py @@ -5,9 +5,9 @@ from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, - PortDistributor, VanillaPostgres, ) +from fixtures.port_distributor import PortDistributor from fixtures.types import Lsn, TimelineId from fixtures.utils import query_scalar, subprocess_capture diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py index e26b1a980e..53fa70903f 100644 --- a/test_runner/regress/test_gc_aggressive.py +++ b/test_runner/regress/test_gc_aggressive.py @@ -8,9 +8,9 @@ from fixtures.neon_fixtures import ( Endpoint, NeonEnv, NeonEnvBuilder, - RemoteStorageKind, wait_for_last_flush_lsn, ) +from fixtures.remote_storage import RemoteStorageKind from fixtures.types import TenantId, TimelineId from fixtures.utils import query_scalar diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index d4f0c94a29..1269210d0d 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -4,10 +4,10 @@ import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, - RemoteStorageKind, wait_for_last_flush_lsn, ) from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload +from fixtures.remote_storage import RemoteStorageKind from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar diff --git a/test_runner/regress/test_metric_collection.py b/test_runner/regress/test_metric_collection.py index 61c8f800f8..80ffe5126d 100644 --- a/test_runner/regress/test_metric_collection.py +++ b/test_runner/regress/test_metric_collection.py @@ -13,11 +13,11 @@ from fixtures.neon_fixtures import ( PSQL, NeonEnvBuilder, NeonProxy, - PortDistributor, - RemoteStorageKind, VanillaPostgres, wait_for_last_flush_lsn, ) +from fixtures.port_distributor import PortDistributor +from fixtures.remote_storage import RemoteStorageKind from fixtures.types import TenantId, TimelineId from fixtures.utils import query_scalar from pytest_httpserver import HTTPServer diff --git a/test_runner/regress/test_neon_local_cli.py b/test_runner/regress/test_neon_local_cli.py index 6dd47de8cf..becdd9ff80 100644 --- a/test_runner/regress/test_neon_local_cli.py +++ b/test_runner/regress/test_neon_local_cli.py @@ -1,4 +1,5 @@ -from fixtures.neon_fixtures import NeonEnvBuilder, PortDistributor +from fixtures.neon_fixtures import NeonEnvBuilder +from fixtures.port_distributor import PortDistributor # Test that neon cli is able to start and stop all processes with the user defaults. diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index ca43df350d..17a63535cf 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -10,8 +10,6 @@ import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, - RemoteStorageKind, - available_remote_storages, last_flush_lsn_upload, wait_for_last_flush_lsn, ) @@ -23,6 +21,7 @@ from fixtures.pageserver.utils import ( wait_for_upload_queue_empty, wait_until_tenant_state, ) +from fixtures.remote_storage import RemoteStorageKind, available_remote_storages from fixtures.types import Lsn from fixtures.utils import query_scalar, wait_until diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 7c04ed9017..68116985d3 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -12,10 +12,7 @@ from typing import Dict, List, Optional, Tuple import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( - LocalFsStorage, NeonEnvBuilder, - RemoteStorageKind, - available_remote_storages, wait_for_last_flush_lsn, ) from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient @@ -26,6 +23,11 @@ from fixtures.pageserver.utils import ( wait_until_tenant_active, wait_until_tenant_state, ) +from fixtures.remote_storage import ( + LocalFsStorage, + RemoteStorageKind, + available_remote_storages, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import print_gc_result, query_scalar, wait_until from requests import ReadTimeout diff --git a/test_runner/regress/test_sni_router.py b/test_runner/regress/test_sni_router.py index 9b78e8287e..4336e6551d 100644 --- a/test_runner/regress/test_sni_router.py +++ b/test_runner/regress/test_sni_router.py @@ -6,7 +6,8 @@ from typing import Optional, Type import backoff from fixtures.log_helper import log -from fixtures.neon_fixtures import PgProtocol, PortDistributor, VanillaPostgres +from fixtures.neon_fixtures import PgProtocol, VanillaPostgres +from fixtures.port_distributor import PortDistributor def generate_tls_cert(cn, certout, keyout): diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 7c80d86863..60ec532db4 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -4,11 +4,10 @@ from contextlib import closing import psycopg2.extras from fixtures.log_helper import log from fixtures.neon_fixtures import ( - LocalFsStorage, NeonEnvBuilder, - RemoteStorageKind, ) from fixtures.pageserver.utils import assert_tenant_state, wait_for_upload +from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.types import Lsn from fixtures.utils import wait_until diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 77f93dbd92..932d821954 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -11,8 +11,6 @@ from fixtures.neon_fixtures import ( Endpoint, NeonEnv, NeonEnvBuilder, - RemoteStorageKind, - available_remote_storages, ) from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.pageserver.utils import ( @@ -20,6 +18,7 @@ from fixtures.pageserver.utils import ( wait_for_upload, wait_until_tenant_state, ) +from fixtures.remote_storage import RemoteStorageKind, available_remote_storages from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until from prometheus_client.samples import Sample diff --git a/test_runner/regress/test_tenant_relocation.py b/test_runner/regress/test_tenant_relocation.py index 2805d56c98..eb020c101f 100644 --- a/test_runner/regress/test_tenant_relocation.py +++ b/test_runner/regress/test_tenant_relocation.py @@ -7,15 +7,12 @@ from pathlib import Path from typing import Any, Dict, Optional, Tuple import pytest +from fixtures.broker import NeonBroker from fixtures.log_helper import log from fixtures.neon_fixtures import ( Endpoint, - NeonBroker, NeonEnv, NeonEnvBuilder, - PortDistributor, - RemoteStorageKind, - available_remote_storages, ) from fixtures.pageserver.http import PageserverHttpClient from fixtures.pageserver.utils import ( @@ -24,6 +21,8 @@ from fixtures.pageserver.utils import ( wait_for_last_record_lsn, wait_for_upload, ) +from fixtures.port_distributor import PortDistributor +from fixtures.remote_storage import RemoteStorageKind, available_remote_storages from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import ( query_scalar, diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index e327910138..19bc3ed37c 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -18,10 +18,9 @@ from fixtures.metrics import ( from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, - RemoteStorageKind, - available_remote_storages, ) from fixtures.pageserver.utils import timeline_delete_wait_completed +from fixtures.remote_storage import RemoteStorageKind, available_remote_storages from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import wait_until from prometheus_client.samples import Sample diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index 498563325b..79a3b353d4 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -16,11 +16,8 @@ import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( Endpoint, - LocalFsStorage, NeonEnv, NeonEnvBuilder, - RemoteStorageKind, - available_remote_storages, last_flush_lsn_upload, ) from fixtures.pageserver.utils import ( @@ -28,6 +25,11 @@ from fixtures.pageserver.utils import ( wait_for_last_record_lsn, wait_for_upload, ) +from fixtures.remote_storage import ( + LocalFsStorage, + RemoteStorageKind, + available_remote_storages, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until diff --git a/test_runner/regress/test_threshold_based_eviction.py b/test_runner/regress/test_threshold_based_eviction.py index 4d8c87e09e..b1bc9623ce 100644 --- a/test_runner/regress/test_threshold_based_eviction.py +++ b/test_runner/regress/test_threshold_based_eviction.py @@ -6,10 +6,10 @@ from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, - RemoteStorageKind, last_flush_lsn_upload, ) from fixtures.pageserver.http import LayerMapInfo +from fixtures.remote_storage import RemoteStorageKind from fixtures.types import TimelineId from pytest_httpserver import HTTPServer diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py index 5ba34122bc..ca737ac02d 100644 --- a/test_runner/regress/test_timeline_delete.py +++ b/test_runner/regress/test_timeline_delete.py @@ -13,9 +13,6 @@ from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, PgBin, - RemoteStorageKind, - S3Storage, - available_remote_storages, last_flush_lsn_upload, wait_for_last_flush_lsn, ) @@ -28,6 +25,11 @@ from fixtures.pageserver.utils import ( wait_until_tenant_active, wait_until_timeline_state, ) +from fixtures.remote_storage import ( + RemoteStorageKind, + S3Storage, + available_remote_storages, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index cb993c93d2..f6e4a667a4 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -16,8 +16,6 @@ from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, PgBin, - PortDistributor, - RemoteStorageKind, VanillaPostgres, wait_for_last_flush_lsn, ) @@ -29,6 +27,8 @@ from fixtures.pageserver.utils import ( wait_until_tenant_active, ) from fixtures.pg_version import PgVersion +from fixtures.port_distributor import PortDistributor +from fixtures.remote_storage import RemoteStorageKind from fixtures.types import TenantId, TimelineId from fixtures.utils import get_timeline_dir_size, wait_until diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 24b32ad7e7..87f076d236 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -15,22 +15,18 @@ from typing import Any, List, Optional import psycopg2 import pytest +from fixtures.broker import NeonBroker from fixtures.log_helper import log from fixtures.neon_fixtures import ( Endpoint, - NeonBroker, NeonEnv, NeonEnvBuilder, NeonPageserver, PgBin, PgProtocol, - PortDistributor, - RemoteStorageKind, - RemoteStorageUsers, Safekeeper, SafekeeperHttpClient, SafekeeperPort, - available_remote_storages, ) from fixtures.pageserver.utils import ( timeline_delete_wait_completed, @@ -38,6 +34,12 @@ from fixtures.pageserver.utils import ( wait_for_upload, ) from fixtures.pg_version import PgVersion +from fixtures.port_distributor import PortDistributor +from fixtures.remote_storage import ( + RemoteStorageKind, + RemoteStorageUsers, + available_remote_storages, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import get_dir_size, query_scalar, start_in_background diff --git a/test_runner/regress/test_wal_restore.py b/test_runner/regress/test_wal_restore.py index f3d3a84c20..c97c69db23 100644 --- a/test_runner/regress/test_wal_restore.py +++ b/test_runner/regress/test_wal_restore.py @@ -5,9 +5,9 @@ import pytest from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, - PortDistributor, VanillaPostgres, ) +from fixtures.port_distributor import PortDistributor from fixtures.types import TenantId, TimelineId