From 1e62eb9ba0f4540edddce8ee80909da59477571f Mon Sep 17 00:00:00 2001 From: Matthias van de Meent Date: Thu, 12 Sep 2024 21:56:00 +0100 Subject: [PATCH] Great Success! with shared fixtures Not everything is done yet, but a good amount of this is done. --- test_runner/fixtures/neon_fixtures.py | 188 +++++++++++++++++++-- test_runner/fixtures/parametrize.py | 64 ++++--- test_runner/regress/test_endpoint_crash.py | 9 +- test_runner/regress/test_fsm_truncate.py | 8 +- test_runner/regress/test_vm_bits.py | 5 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- 7 files changed, 226 insertions(+), 52 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 60887b9aed..c95cbd2353 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -138,7 +138,7 @@ def base_dir() -> Iterator[Path]: yield base_dir -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def neon_binpath(base_dir: Path, build_type: str) -> Iterator[Path]: if os.getenv("REMOTE_ENV"): # we are in remote env and do not have neon binaries locally @@ -158,7 +158,7 @@ def neon_binpath(base_dir: Path, build_type: str) -> Iterator[Path]: yield binpath -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def pg_distrib_dir(base_dir: Path) -> Iterator[Path]: if env_postgres_bin := os.environ.get("POSTGRES_DISTRIB_DIR"): distrib_dir = Path(env_postgres_bin).resolve() @@ -182,7 +182,7 @@ def top_output_dir(base_dir: Path) -> Iterator[Path]: yield output_dir -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def versioned_pg_distrib_dir(pg_distrib_dir: Path, pg_version: PgVersion) -> Iterator[Path]: versioned_dir = pg_distrib_dir / pg_version.v_prefixed @@ -272,6 +272,20 @@ def default_broker( yield broker broker.stop() +@pytest.fixture(scope="session") +def shared_broker( + port_distributor: PortDistributor, + shared_test_output_dir: Path, + neon_binpath: Path, +) -> Iterator[NeonBroker]: + # multiple pytest sessions could get launched in parallel, get them different ports/datadirs + client_port = port_distributor.get_port() + broker_logfile = shared_test_output_dir / "repo" / "storage_broker.log" + + broker = NeonBroker(logfile=broker_logfile, port=client_port, neon_binpath=neon_binpath) + yield broker + broker.stop() + @pytest.fixture(scope="session") def run_id() -> Iterator[uuid.UUID]: @@ -484,6 +498,7 @@ class NeonEnvBuilder: safekeeper_extra_opts: Optional[list[str]] = None, storage_controller_port_override: Optional[int] = None, pageserver_io_buffer_alignment: Optional[int] = None, + shared: Optional[bool] = False, ): self.repo_dir = repo_dir self.rust_log_override = rust_log_override @@ -542,7 +557,7 @@ class NeonEnvBuilder: assert test_name.startswith( "test_" - ), "Unexpectedly instantiated from outside a test function" + ) or shared, "Unexpectedly instantiated from outside a test function" self.test_name = test_name def init_configs(self, default_remote_storage_if_missing: bool = True) -> NeonEnv: @@ -1403,6 +1418,9 @@ class NeonEnv: return "ep-" + str(self.endpoint_counter) + + + @pytest.fixture(scope="function") def neon_simple_env( request: FixtureRequest, @@ -1452,6 +1470,73 @@ def neon_simple_env( yield env +@pytest.fixture(scope="session", autouse=True) +def neon_shared_env( + pytestconfig: Config, + port_distributor: PortDistributor, + mock_s3_server: MockS3Server, + shared_broker: NeonBroker, + run_id: uuid.UUID, + top_output_dir: Path, + shared_test_output_dir: Path, + neon_binpath: Path, + build_type: str, + pg_distrib_dir: Path, + pg_version: PgVersion, + pageserver_virtual_file_io_engine: str, + pageserver_aux_file_policy: Optional[AuxFileStore], + pageserver_default_tenant_config_compaction_algorithm: Optional[Dict[str, Any]], + pageserver_io_buffer_alignment: Optional[int], + request: FixtureRequest, +) -> Iterator[NeonEnv]: + """ + Simple Neon environment, with no authentication and no safekeepers. + + This fixture will use RemoteStorageKind.LOCAL_FS with pageserver. + """ + + prefix = f"shared[{build_type}-{pg_version.v_prefixed}]-" + + # Create the environment in the per-test output directory + repo_dir = get_test_repo_dir(request, top_output_dir, prefix) + + with NeonEnvBuilder( + top_output_dir=top_output_dir, + repo_dir=repo_dir, + port_distributor=port_distributor, + broker=shared_broker, + mock_s3_server=mock_s3_server, + neon_binpath=neon_binpath, + pg_distrib_dir=pg_distrib_dir, + pg_version=pg_version, + run_id=run_id, + preserve_database_files=cast(bool, pytestconfig.getoption("--preserve-database-files")), + test_name=f"{prefix}{request.node.name}", + test_output_dir=shared_test_output_dir, + pageserver_virtual_file_io_engine=pageserver_virtual_file_io_engine, + pageserver_aux_file_policy=pageserver_aux_file_policy, + pageserver_default_tenant_config_compaction_algorithm=pageserver_default_tenant_config_compaction_algorithm, + pageserver_io_buffer_alignment=pageserver_io_buffer_alignment, + shared=True, + ) as builder: + env = builder.init_start() + + yield env + + +@pytest.fixture(scope="function") +def neon_endpoint(request: FixtureRequest, neon_shared_env: NeonEnv) -> Endpoint: + neon_shared_env.neon_cli.create_branch(request.node.name) + ep = neon_shared_env.endpoints.create_start(request.node.name) + + try: + yield ep + finally: + if ep.is_running(): + try: + ep.stop() + except BaseException: + pass @pytest.fixture(scope="function") def neon_env_builder( @@ -4808,27 +4893,27 @@ def _get_test_dir(request: FixtureRequest, top_output_dir: Path, prefix: str) -> return test_dir -def get_test_output_dir(request: FixtureRequest, top_output_dir: Path) -> Path: +def get_test_output_dir(request: FixtureRequest, top_output_dir: Path, prefix: Optional[str] = None) -> Path: """ The working directory for a test. """ - return _get_test_dir(request, top_output_dir, "") + return _get_test_dir(request, top_output_dir, prefix or "") -def get_test_overlay_dir(request: FixtureRequest, top_output_dir: Path) -> Path: +def get_test_overlay_dir(request: FixtureRequest, top_output_dir: Path, prefix: Optional[str] = None) -> Path: """ Directory that contains `upperdir` and `workdir` for overlayfs mounts that a test creates. See `NeonEnvBuilder.overlay_mount`. """ - return _get_test_dir(request, top_output_dir, "overlay-") + return _get_test_dir(request, top_output_dir, f"overlay-{prefix or ''}") -def get_shared_snapshot_dir_path(top_output_dir: Path, snapshot_name: str) -> Path: - return top_output_dir / "shared-snapshots" / snapshot_name +def get_shared_snapshot_dir_path(top_output_dir: Path, snapshot_name: str, prefix: Optional[str] = None) -> Path: + return top_output_dir / f"{prefix or ''}shared-snapshots" / snapshot_name -def get_test_repo_dir(request: FixtureRequest, top_output_dir: Path) -> Path: - return get_test_output_dir(request, top_output_dir) / "repo" +def get_test_repo_dir(request: FixtureRequest, top_output_dir: Path, prefix: Optional[str] = None) -> Path: + return get_test_output_dir(request, top_output_dir, prefix or '') / "repo" def pytest_addoption(parser: Parser): @@ -4881,6 +4966,49 @@ def test_output_dir( allure_attach_from_dir(test_dir, preserve_database_files) +# This is autouse, so the test output directory always gets created, even +# if a test doesn't put anything there. +# +# NB: we request the overlay dir fixture so the fixture does its cleanups +@pytest.fixture(scope="session", autouse=True) +def shared_test_output_dir( + request: FixtureRequest, + pg_version: PgVersion, + build_type: str, + top_output_dir: Path, + shared_test_overlay_dir: Path +) -> Iterator[Path]: + """Create the working directory for shared tests.""" + + prefix = f"shared[{build_type}-{pg_version.v_prefixed}]-" + + # one directory per test + test_dir = get_test_output_dir(request, top_output_dir, prefix) + + log.info(f"test_output_dir is {test_dir}") + shutil.rmtree(test_dir, ignore_errors=True) + test_dir.mkdir() + + yield test_dir + + # Allure artifacts creation might involve the creation of `.tar.zst` archives, + # which aren't going to be used if Allure results collection is not enabled + # (i.e. --alluredir is not set). + # Skip `allure_attach_from_dir` in this case + if not request.config.getoption("--alluredir"): + return + + preserve_database_files = False + for k, v in request.node.user_properties: + # NB: the neon_env_builder fixture uses this fixture (test_output_dir). + # So, neon_env_builder's cleanup runs before here. + # The cleanup propagates NeonEnvBuilder.preserve_database_files into this user property. + if k == "preserve_database_files": + assert isinstance(v, bool) + preserve_database_files = v + + allure_attach_from_dir(test_dir, preserve_database_files) + class FileAndThreadLock: def __init__(self, path: Path): @@ -4987,6 +5115,42 @@ def test_overlay_dir(request: FixtureRequest, top_output_dir: Path) -> Optional[ # and on unclean shutdown, this function will take care of it # on the next test run +@pytest.fixture(scope="session") +def shared_test_overlay_dir(request: FixtureRequest, top_output_dir: Path) -> Optional[Path]: + """ + Idempotently create a test's overlayfs mount state directory. + If the functionality isn't enabled via env var, returns None. + + The procedure cleans up after previous runs that were aborted (e.g. due to Ctrl-C, OOM kills, etc). + """ + + if os.getenv("NEON_ENV_BUILDER_USE_OVERLAYFS_FOR_SNAPSHOTS") is None: + return None + + overlay_dir = get_test_overlay_dir(request, top_output_dir) + log.info(f"test_overlay_dir is {overlay_dir}") + + overlay_dir.mkdir(exist_ok=True) + # unmount stale overlayfs mounts which subdirectories of `overlay_dir/*` as the overlayfs `upperdir` and `workdir` + for mountpoint in overlayfs.iter_mounts_beneath(get_test_output_dir(request, top_output_dir)): + cmd = ["sudo", "umount", str(mountpoint)] + log.info( + f"Unmounting stale overlayfs mount probably created during earlier test run: {cmd}" + ) + subprocess.run(cmd, capture_output=True, check=True) + # the overlayfs `workdir`` is owned by `root`, shutil.rmtree won't work. + cmd = ["sudo", "rm", "-rf", str(overlay_dir)] + subprocess.run(cmd, capture_output=True, check=True) + + overlay_dir.mkdir() + + return overlay_dir + + # no need to clean up anything: on clean shutdown, + # NeonEnvBuilder.overlay_cleanup_teardown takes care of cleanup + # and on unclean shutdown, this function will take care of it + # on the next test run + SKIP_DIRS = frozenset( ( diff --git a/test_runner/fixtures/parametrize.py b/test_runner/fixtures/parametrize.py index e2dd51802c..094d0bf2fb 100644 --- a/test_runner/fixtures/parametrize.py +++ b/test_runner/fixtures/parametrize.py @@ -13,33 +13,60 @@ from fixtures.utils import AuxFileStore Dynamically parametrize tests by different parameters """ +def get_pgversions(): + if (v := os.getenv("DEFAULT_PG_VERSION")) is None: + pg_versions = [version for version in PgVersion if version != PgVersion.NOT_SET] + else: + pg_versions = [PgVersion(v)] -@pytest.fixture(scope="function", autouse=True) -def pg_version() -> Optional[PgVersion]: - return None + return pg_versions -@pytest.fixture(scope="function", autouse=True) -def build_type() -> Optional[str]: - return None +@pytest.fixture( + scope="session", + autouse=True, + params=get_pgversions(), + ids=lambda v: f"pg{v}", +) +def pg_version(request) -> Optional[PgVersion]: + return request.param -@pytest.fixture(scope="function", autouse=True) +def get_buildtypes(): + if (bt := os.getenv("BUILD_TYPE")) is None: + build_types = ["debug", "release"] + else: + build_types = [bt.lower()] + + return build_types + + +@pytest.fixture( + scope="session", + autouse=True, + params=get_buildtypes(), + ids=lambda t: f"{t}", +) +def build_type(request) -> Optional[str]: + return request.param + + +@pytest.fixture(scope="session", autouse=True) def platform() -> Optional[str]: return None -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="session", autouse=True) def pageserver_virtual_file_io_engine() -> Optional[str]: return os.getenv("PAGESERVER_VIRTUAL_FILE_IO_ENGINE") -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="session", autouse=True) def pageserver_io_buffer_alignment() -> Optional[int]: return None -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="session", autouse=True) def pageserver_aux_file_policy() -> Optional[AuxFileStore]: return None @@ -53,26 +80,12 @@ def get_pageserver_default_tenant_config_compaction_algorithm() -> Optional[Dict return v -@pytest.fixture(scope="function", autouse=True) +@pytest.fixture(scope="session", autouse=True) def pageserver_default_tenant_config_compaction_algorithm() -> Optional[Dict[str, Any]]: return get_pageserver_default_tenant_config_compaction_algorithm() def pytest_generate_tests(metafunc: Metafunc): - if (bt := os.getenv("BUILD_TYPE")) is None: - build_types = ["debug", "release"] - else: - build_types = [bt.lower()] - - metafunc.parametrize("build_type", build_types) - - if (v := os.getenv("DEFAULT_PG_VERSION")) is None: - pg_versions = [version for version in PgVersion if version != PgVersion.NOT_SET] - else: - pg_versions = [PgVersion(v)] - - metafunc.parametrize("pg_version", pg_versions, ids=map(lambda v: f"pg{v}", pg_versions)) - # A hacky way to parametrize tests only for `pageserver_virtual_file_io_engine=std-fs` # And do not change test name for default `pageserver_virtual_file_io_engine=tokio-epoll-uring` to keep tests statistics if (io_engine := os.getenv("PAGESERVER_VIRTUAL_FILE_IO_ENGINE", "")) not in ( @@ -89,6 +102,7 @@ def pytest_generate_tests(metafunc: Metafunc): "pageserver_default_tenant_config_compaction_algorithm", [explicit_default], ids=[explicit_default["kind"]], + scope="session", ) # For performance tests, parametrize also by platform diff --git a/test_runner/regress/test_endpoint_crash.py b/test_runner/regress/test_endpoint_crash.py index ae3dded437..7a432b4632 100644 --- a/test_runner/regress/test_endpoint_crash.py +++ b/test_runner/regress/test_endpoint_crash.py @@ -1,5 +1,5 @@ import pytest -from fixtures.neon_fixtures import NeonEnvBuilder +from fixtures.neon_fixtures import Endpoint @pytest.mark.parametrize( @@ -10,14 +10,11 @@ from fixtures.neon_fixtures import NeonEnvBuilder "💣", # calls `trigger_segfault` internally ], ) -def test_endpoint_crash(neon_env_builder: NeonEnvBuilder, sql_func: str): +def test_endpoint_crash(neon_endpoint: Endpoint, sql_func: str): """ Test that triggering crash from neon_test_utils crashes the endpoint """ - env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_endpoint_crash") - endpoint = env.endpoints.create_start("test_endpoint_crash") - + endpoint = neon_endpoint endpoint.safe_psql("CREATE EXTENSION neon_test_utils;") with pytest.raises(Exception, match="This probably means the server terminated abnormally"): endpoint.safe_psql(f"SELECT {sql_func}();") diff --git a/test_runner/regress/test_fsm_truncate.py b/test_runner/regress/test_fsm_truncate.py index 80e4da8380..5b20176943 100644 --- a/test_runner/regress/test_fsm_truncate.py +++ b/test_runner/regress/test_fsm_truncate.py @@ -1,10 +1,8 @@ -from fixtures.neon_fixtures import NeonEnvBuilder +from fixtures.neon_fixtures import Endpoint -def test_fsm_truncate(neon_env_builder: NeonEnvBuilder): - env = neon_env_builder.init_start() - env.neon_cli.create_branch("test_fsm_truncate") - endpoint = env.endpoints.create_start("test_fsm_truncate") +def test_fsm_truncate(neon_endpoint: Endpoint): + endpoint = neon_endpoint endpoint.safe_psql( "CREATE TABLE t1(key int); CREATE TABLE t2(key int); TRUNCATE TABLE t1; TRUNCATE TABLE t2;" ) diff --git a/test_runner/regress/test_vm_bits.py b/test_runner/regress/test_vm_bits.py index 3075211ada..90c20d808a 100644 --- a/test_runner/regress/test_vm_bits.py +++ b/test_runner/regress/test_vm_bits.py @@ -2,7 +2,7 @@ import time from contextlib import closing from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, fork_at_current_lsn +from fixtures.neon_fixtures import Endpoint, NeonEnv, NeonEnvBuilder, fork_at_current_lsn from fixtures.utils import query_scalar @@ -57,7 +57,7 @@ def test_vm_bit_clear(neon_simple_env: NeonEnv): cur.execute("UPDATE vmtest_cold_update2 SET id = 5000, filler=repeat('x', 200) WHERE id = 1") # Branch at this point, to test that later - fork_at_current_lsn(env, endpoint, "test_vm_bit_clear_new", "main") + # fork_at_current_lsn(env, endpoint, "test_vm_bit_clear_new", "main") # Clear the buffer cache, to force the VM page to be re-fetched from # the page server @@ -91,6 +91,7 @@ def test_vm_bit_clear(neon_simple_env: NeonEnv): # a dirty VM page is evicted. If the VM bit was not correctly cleared by the # earlier WAL record, the full-page image hides the problem. Starting a new # server at the right point-in-time avoids that full-page image. + endpoint_new = env.endpoints.create_start("test_vm_bit_clear_new") pg_new_conn = endpoint_new.connect() diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 49d5e576a5..0353a8f5f7 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 49d5e576a56e4cc59cd6a6a0791b2324b9fa675e +Subproject commit 0353a8f5f7eabd73450af7e483f33763480878fa diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 6e9a4ff624..49386938eb 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 6e9a4ff6249ac02b8175054b7b3f7dfb198be48b +Subproject commit 49386938ebdeeef7a2b2afc212c735239cdb87dc