mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 21:42:56 +00:00
bee04b8a69
## Problem
We need a way to incrementally switch to direct IO. During the rollout
we might want to switch to O_DIRECT on image and delta layer read path
first before others.
## Summary of changes
- Revisited and simplified direct io config in `PageserverConf`.
- We could add a fallback mode for open, but for read there isn't a
reasonable alternative (without creating another buffered virtual file).
- Added a wrapper around `VirtualFile`, current implementation become
`VirtualFileInner`
- Use `open_v2`, `create_v2`, `open_with_options_v2` when we want to use
the IO mode specified in PS config.
- Once we onboard all IO through VirtualFile using this new API, we will
delete the old code path.
- Make io mode live configurable for benchmarking.
- Only guaranteed for files opened after the config change, so do it
before the experiment.
As an example, we are using `open_v2` with
`virtual_file::IoMode::Direct` in
https://github.com/neondatabase/neon/pull/9169
We also remove `io_buffer_alignment` config in
a04cfd754b and use it as a compile time
constant. This way we don't have to carry the alignment around or make
frequent call to retrieve this information from the static variable.
Signed-off-by: Yuchen Liang <yuchen@neon.tech>
126 lines
3.9 KiB
Python
126 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
from typing import TYPE_CHECKING
|
|
|
|
import allure
|
|
import pytest
|
|
import toml
|
|
from _pytest.python import Metafunc
|
|
|
|
from fixtures.pg_version import PgVersion
|
|
from fixtures.utils import AuxFileStore
|
|
|
|
if TYPE_CHECKING:
|
|
from typing import Any, Optional
|
|
|
|
|
|
"""
|
|
Dynamically parametrize tests by different parameters
|
|
"""
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def pg_version() -> Optional[PgVersion]:
|
|
return None
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def build_type() -> Optional[str]:
|
|
return None
|
|
|
|
|
|
@pytest.fixture(scope="session", autouse=True)
|
|
def platform() -> Optional[str]:
|
|
return None
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def pageserver_virtual_file_io_engine() -> Optional[str]:
|
|
return os.getenv("PAGESERVER_VIRTUAL_FILE_IO_ENGINE")
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def pageserver_virtual_file_io_mode() -> Optional[str]:
|
|
return os.getenv("PAGESERVER_VIRTUAL_FILE_IO_MODE")
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def pageserver_aux_file_policy() -> Optional[AuxFileStore]:
|
|
return None
|
|
|
|
|
|
def get_pageserver_default_tenant_config_compaction_algorithm() -> Optional[dict[str, Any]]:
|
|
toml_table = os.getenv("PAGESERVER_DEFAULT_TENANT_CONFIG_COMPACTION_ALGORITHM")
|
|
if toml_table is None:
|
|
return None
|
|
v = toml.loads(toml_table)
|
|
assert isinstance(v, dict)
|
|
return v
|
|
|
|
|
|
@pytest.fixture(scope="function", autouse=True)
|
|
def pageserver_default_tenant_config_compaction_algorithm() -> Optional[dict[str, Any]]:
|
|
return get_pageserver_default_tenant_config_compaction_algorithm()
|
|
|
|
|
|
def pytest_generate_tests(metafunc: Metafunc):
|
|
if (bt := os.getenv("BUILD_TYPE")) is None:
|
|
build_types = ["debug", "release"]
|
|
else:
|
|
build_types = [bt.lower()]
|
|
|
|
metafunc.parametrize("build_type", build_types)
|
|
|
|
if (v := os.getenv("DEFAULT_PG_VERSION")) is None:
|
|
pg_versions = [version for version in PgVersion if version != PgVersion.NOT_SET]
|
|
else:
|
|
pg_versions = [PgVersion(v)]
|
|
|
|
metafunc.parametrize("pg_version", pg_versions, ids=map(lambda v: f"pg{v}", pg_versions))
|
|
|
|
# A hacky way to parametrize tests only for `pageserver_virtual_file_io_engine=std-fs`
|
|
# And do not change test name for default `pageserver_virtual_file_io_engine=tokio-epoll-uring` to keep tests statistics
|
|
if (io_engine := os.getenv("PAGESERVER_VIRTUAL_FILE_IO_ENGINE", "")) not in (
|
|
"",
|
|
"tokio-epoll-uring",
|
|
):
|
|
metafunc.parametrize("pageserver_virtual_file_io_engine", [io_engine])
|
|
|
|
# Same hack for pageserver_default_tenant_config_compaction_algorithm
|
|
if (
|
|
explicit_default := get_pageserver_default_tenant_config_compaction_algorithm()
|
|
) is not None:
|
|
metafunc.parametrize(
|
|
"pageserver_default_tenant_config_compaction_algorithm",
|
|
[explicit_default],
|
|
ids=[explicit_default["kind"]],
|
|
)
|
|
|
|
# For performance tests, parametrize also by platform
|
|
if (
|
|
"test_runner/performance" in metafunc.definition._nodeid
|
|
and (platform := os.getenv("PLATFORM")) is not None
|
|
):
|
|
metafunc.parametrize("platform", [platform.lower()])
|
|
|
|
|
|
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
|
|
def pytest_runtest_makereport(*args, **kwargs):
|
|
# Add test parameters to Allue report to distinguish the same tests with different parameters.
|
|
# Names has `__` prefix to avoid conflicts with `pytest.mark.parametrize` parameters
|
|
|
|
# A mapping between `uname -m` and `RUNNER_ARCH` values.
|
|
# `RUNNER_ARCH` environment variable is set on GitHub Runners,
|
|
# possible values are X86, X64, ARM, or ARM64.
|
|
# See https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables
|
|
uname_m = {
|
|
"aarch64": "ARM64",
|
|
"arm64": "ARM64",
|
|
"x86_64": "X64",
|
|
}.get(os.uname().machine, "UNKNOWN")
|
|
arch = os.getenv("RUNNER_ARCH", uname_m)
|
|
allure.dynamic.parameter("__arch", arch)
|
|
|
|
yield
|