diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js index 432c78d1af..dd60d42a37 100755 --- a/scripts/comment-test-report.js +++ b/scripts/comment-test-report.js @@ -144,17 +144,24 @@ const reportSummary = async (params) => { } summary += `- \`${testName}\`: ${links.join(", ")}\n` } - - const testsToRerun = Object.values(failedTests[pgVersion]).map(x => x[0].name) - const command = `DEFAULT_PG_VERSION=${pgVersion} scripts/pytest -k "${testsToRerun.join(" or ")}"` - - summary += "```\n" - summary += `# Run failed on Postgres ${pgVersion} tests locally:\n` - summary += `${command}\n` - summary += "```\n" } } + if (failedTestsCount > 0) { + const testsToRerun = [] + for (const pgVersion of Object.keys(failedTests)) { + for (const testName of Object.keys(failedTests[pgVersion])) { + testsToRerun.push(...failedTests[pgVersion][testName].map(test => test.name)) + } + } + const command = `scripts/pytest -vv -n $(nproc) -k "${testsToRerun.join(" or ")}"` + + summary += "```\n" + summary += `# Run all failed tests locally:\n` + summary += `${command}\n` + summary += "```\n" + } + if (flakyTestsCount > 0) { summary += `
\nFlaky tests (${flakyTestsCount})\n\n` for (const pgVersion of Array.from(pgVersions).sort().reverse()) { @@ -164,8 +171,7 @@ const reportSummary = async (params) => { const links = [] for (const test of tests) { const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}/retries` - const status = test.status === "passed" ? ":white_check_mark:" : ":x:" - links.push(`[${status} ${test.buildType}](${allureLink})`) + links.push(`[${test.buildType}](${allureLink})`) } summary += `- \`${testName}\`: ${links.join(", ")}\n` } diff --git a/test_runner/conftest.py b/test_runner/conftest.py index 4e649e111a..1c36c1ed02 100644 --- a/test_runner/conftest.py +++ b/test_runner/conftest.py @@ -1,6 +1,6 @@ pytest_plugins = ( "fixtures.pg_version", - "fixtures.allure", + "fixtures.parametrize", "fixtures.neon_fixtures", "fixtures.benchmark_fixture", "fixtures.pg_stats", diff --git a/test_runner/fixtures/allure.py b/test_runner/fixtures/allure.py deleted file mode 100644 index 6f40bd2aa2..0000000000 --- a/test_runner/fixtures/allure.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -import pytest - -from fixtures.pg_version import DEFAULT_VERSION, PgVersion - -""" -Set of utilities to make Allure report more informative. - -- It adds BUILD_TYPE and DEFAULT_PG_VERSION to the test names (only in test_runner/regress) -to make tests distinguishable in Allure report. -""" - - -@pytest.fixture(scope="function", autouse=True) -def allure_noop(): - pass - - -def pytest_generate_tests(metafunc): - if "test_runner/regress" in metafunc.definition._nodeid: - build_type = os.environ.get("BUILD_TYPE", "DEBUG").lower() - pg_version = PgVersion(os.environ.get("DEFAULT_PG_VERSION", DEFAULT_VERSION)) - - metafunc.parametrize("allure_noop", [f"{build_type}-pg{pg_version}"]) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index e56bf78019..c3e9853978 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -102,8 +102,8 @@ def base_dir() -> Iterator[Path]: yield base_dir -@pytest.fixture(scope="session") -def neon_binpath(base_dir: Path) -> Iterator[Path]: +@pytest.fixture(scope="function") +def neon_binpath(base_dir: Path, build_type: str) -> Iterator[Path]: if os.getenv("REMOTE_ENV"): # we are in remote env and do not have neon binaries locally # this is the case for benchmarks run on self-hosted runner @@ -113,7 +113,6 @@ def neon_binpath(base_dir: Path) -> Iterator[Path]: if env_neon_bin := os.environ.get("NEON_BIN"): binpath = Path(env_neon_bin) else: - build_type = os.environ.get("BUILD_TYPE", "debug") binpath = base_dir / "target" / build_type log.info(f"neon_binpath is {binpath}") @@ -123,7 +122,7 @@ def neon_binpath(base_dir: Path) -> Iterator[Path]: yield binpath -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def pg_distrib_dir(base_dir: Path) -> Iterator[Path]: if env_postgres_bin := os.environ.get("POSTGRES_DISTRIB_DIR"): distrib_dir = Path(env_postgres_bin).resolve() @@ -147,7 +146,7 @@ def top_output_dir(base_dir: Path) -> Iterator[Path]: yield output_dir -@pytest.fixture(scope="session") +@pytest.fixture(scope="function") def versioned_pg_distrib_dir(pg_distrib_dir: Path, pg_version: PgVersion) -> Iterator[Path]: versioned_dir = pg_distrib_dir / pg_version.v_prefixed @@ -174,7 +173,23 @@ def shareable_scope(fixture_name: str, config: Config) -> Literal["session", "fu def myfixture(...) ... """ - return "function" if os.environ.get("TEST_SHARED_FIXTURES") is None else "session" + scope: Literal["session", "function"] + + if os.environ.get("TEST_SHARED_FIXTURES") is None: + # Create the environment in the per-test output directory + scope = "function" + elif ( + os.environ.get("BUILD_TYPE") is not None + and os.environ.get("DEFAULT_PG_VERSION") is not None + ): + scope = "session" + else: + pytest.fail( + "Shared environment(TEST_SHARED_FIXTURES) requires BUILD_TYPE and DEFAULT_PG_VERSION to be set", + pytrace=False, + ) + + return scope @pytest.fixture(scope="session") diff --git a/test_runner/fixtures/parametrize.py b/test_runner/fixtures/parametrize.py new file mode 100644 index 0000000000..53350138dd --- /dev/null +++ b/test_runner/fixtures/parametrize.py @@ -0,0 +1,50 @@ +import os +from typing import Optional + +import pytest +from _pytest.fixtures import FixtureRequest +from _pytest.python import Metafunc + +from fixtures.pg_version import PgVersion + +""" +Dynamically parametrize tests by Postgres version and build type (debug/release/remote) +""" + + +@pytest.fixture(scope="function", autouse=True) +def pg_version(request: FixtureRequest) -> Optional[PgVersion]: + # Do not parametrize performance tests yet, we need to prepare grafana charts first + if "test_runner/performance" in str(request.node.path): + v = os.environ.get("DEFAULT_PG_VERSION") + return PgVersion(v) + + return None + + +@pytest.fixture(scope="function", autouse=True) +def build_type(request: FixtureRequest) -> Optional[str]: + # Do not parametrize performance tests yet, we need to prepare grafana charts first + if "test_runner/performance" in str(request.node.path): + return os.environ.get("BUILD_TYPE", "").lower() + + return None + + +def pytest_generate_tests(metafunc: Metafunc): + # Do not parametrize performance tests yet, we need to prepare grafana charts first + if "test_runner/performance" in metafunc.definition._nodeid: + return + + if (v := os.environ.get("DEFAULT_PG_VERSION")) is None: + pg_versions = [version for version in PgVersion if version != PgVersion.NOT_SET] + else: + pg_versions = [PgVersion(v)] + + if (bt := os.environ.get("BUILD_TYPE")) is None: + build_types = ["debug", "release"] + else: + build_types = [bt.lower()] + + metafunc.parametrize("build_type", build_types) + metafunc.parametrize("pg_version", pg_versions, ids=map(lambda v: f"pg{v}", pg_versions)) diff --git a/test_runner/fixtures/pg_version.py b/test_runner/fixtures/pg_version.py index 14ae88cc2c..b61f52be3c 100644 --- a/test_runner/fixtures/pg_version.py +++ b/test_runner/fixtures/pg_version.py @@ -1,12 +1,10 @@ import enum import os -from typing import Iterator, Optional +from typing import Optional import pytest +from _pytest.config import Config from _pytest.config.argparsing import Parser -from pytest import FixtureRequest - -from fixtures.log_helper import log """ This fixture is used to determine which version of Postgres to use for tests. @@ -75,18 +73,10 @@ def pytest_addoption(parser: Parser): "--pg-version", action="store", type=PgVersion, - help="Postgres version to use for tests", + help="DEPRECATED: Postgres version to use for tests", ) -@pytest.fixture(scope="session") -def pg_version(request: FixtureRequest) -> Iterator[PgVersion]: - if v := request.config.getoption("--pg-version"): - version, source = v, "from --pg-version command-line argument" - elif v := os.environ.get("DEFAULT_PG_VERSION"): - version, source = PgVersion(v), "from DEFAULT_PG_VERSION environment variable" - else: - version, source = DEFAULT_VERSION, "default version" - - log.info(f"pg_version is {version} ({source})") - yield version +def pytest_configure(config: Config): + if config.getoption("--pg-version"): + raise Exception("--pg-version is deprecated, use DEFAULT_PG_VERSION env var instead")