rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-14 00:42:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

use pg_version in python tests

Browse Source
This commit is contained in:
Anastasia Lubennikova
2022-09-18 22:45:29 +03:00
parent fc08062a13
commit cbf655a346
6 changed files with 109 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
scripts/export_import_between_pageservers.py
View File

@@ -80,11 +80,13 @@ def subprocess_capture(capture_dir: str, cmd: List[str], **kwargs: Any) -> str:
class PgBin:
"""A helper class for executing postgres binaries"""
def __init__(self, log_dir: Path, pg_distrib_dir):
def __init__(self, log_dir: Path, pg_distrib_dir, pg_version):
self.log_dir = log_dir
self.pg_bin_path = os.path.join(str(pg_distrib_dir), "bin")
self.pg_bin_path = os.path.join(str(pg_distrib_dir), "v{}".format(pg_version), "bin")
self.env = os.environ.copy()
self.env["LD_LIBRARY_PATH"] = os.path.join(str(pg_distrib_dir), "lib")
self.env["LD_LIBRARY_PATH"] = os.path.join(
str(pg_distrib_dir), "v{}".format(pg_version), "lib"
)
def _fixpath(self, command: List[str]):
if "/" not in command[0]:
@@ -484,7 +486,7 @@ def import_timeline(
with open(stdout_filename, "w") as stdout_f:
with open(stderr_filename2, "w") as stderr_f:
print(f"(capturing output to {stdout_filename})")
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir)
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir, pg_version)
subprocess.run(
full_cmd,
stdout=stdout_f,
@@ -503,7 +505,15 @@ def import_timeline(
def export_timeline(
args, psql_path, pageserver_connstr, tenant_id, timeline_id, last_lsn, prev_lsn, tar_filename
args,
psql_path,
pageserver_connstr,
tenant_id,
timeline_id,
last_lsn,
prev_lsn,
tar_filename,
pg_version,
):
# Choose filenames
incomplete_filename = tar_filename + ".incomplete"
@@ -518,13 +528,13 @@ def export_timeline(
with open(incomplete_filename, "w") as stdout_f:
with open(stderr_filename, "w") as stderr_f:
print(f"(capturing output to {incomplete_filename})")
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir)
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir, pg_version)
subprocess.run(
cmd, stdout=stdout_f, stderr=stderr_f, env=pg_bin._build_env(None), check=True
)
# Add missing rels
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir)
pg_bin = PgBin(args.work_dir, args.pg_distrib_dir, pg_version)
add_missing_rels(incomplete_filename, tar_filename, args.work_dir, pg_bin)
# Log more info
@@ -533,7 +543,8 @@ def export_timeline(
def main(args: argparse.Namespace):
psql_path = str(Path(args.pg_distrib_dir) / "bin" / "psql")
# any psql version will do here. use current DEFAULT_PG_VERSION = 14
psql_path = str(Path(args.pg_distrib_dir) / "v14" / "bin" / "psql")
old_pageserver_host = args.old_pageserver_host
new_pageserver_host = args.new_pageserver_host
@@ -566,6 +577,8 @@ def main(args: argparse.Namespace):
args.work_dir, f"{timeline['tenant_id']}_{timeline['timeline_id']}.tar"
)
pg_version = timeline["local"]["pg_version"]
# Export timeline from old pageserver
if args.only_import is False:
last_lsn, prev_lsn = get_rlsn(
@@ -582,6 +595,7 @@ def main(args: argparse.Namespace):
last_lsn,
prev_lsn,
tar_filename,
pg_version,
)
# Import into new pageserver
@@ -595,7 +609,7 @@ def main(args: argparse.Namespace):
last_lsn,
prev_lsn,
tar_filename,
timeline["local"]["pg_version"],
pg_version,
)
# Re-export and compare
@@ -609,6 +623,7 @@ def main(args: argparse.Namespace):
last_lsn,
prev_lsn,
re_export_filename,
pg_version,
)
# Check the size is the same

106
test_runner/fixtures/neon_fixtures.py
View File

@@ -59,8 +59,8 @@ Env = Dict[str, str]
Fn = TypeVar("Fn", bound=Callable[..., Any])
DEFAULT_OUTPUT_DIR = "test_output"
DEFAULT_PG_VERSION_DEFAULT = "14"
DEFAULT_BRANCH_NAME = "main"
DEFAULT_PG_VERSION_DEFAULT = "14"
BASE_PORT = 15000
WORKER_PORT_NUM = 1000
@@ -71,7 +71,7 @@ base_dir = ""
neon_binpath = ""
pg_distrib_dir = ""
top_output_dir = ""
pg_version = ""
default_pg_version = ""
def pytest_configure(config):
@@ -101,29 +101,36 @@ def pytest_configure(config):
Path(top_output_dir).mkdir(exist_ok=True)
# Find the postgres installation.
global pg_version
pg_version = os.environ.get("DEFAULT_PG_VERSION", DEFAULT_PG_VERSION_DEFAULT)
global default_pg_version
log.info(f"default_pg_version is {default_pg_version}")
env_default_pg_version = os.environ.get("DEFAULT_PG_VERSION")
if env_default_pg_version:
default_pg_version = env_default_pg_version
log.info(f"default_pg_version is set to {default_pg_version}")
else:
default_pg_version = DEFAULT_PG_VERSION_DEFAULT
global pg_distrib_dir
# TODO get rid of the POSTGRES_DISTRIB_DIR env var ?
# use DEFAULT_PG_VERSION instead to generate the path
env_postgres_bin = os.environ.get("POSTGRES_DISTRIB_DIR")
if env_postgres_bin:
pg_distrib_dir = env_postgres_bin
else:
pg_distrib_dir = os.path.normpath(
os.path.join(base_dir, "pg_install/v{}".format(pg_version))
)
pg_distrib_dir = os.path.normpath(os.path.join(base_dir, "pg_install"))
log.info(f"pg_distrib_dir is {pg_distrib_dir}")
psql_bin_path = os.path.join(pg_distrib_dir, "v{}".format(default_pg_version), "bin/psql")
postgres_bin_path = os.path.join(
pg_distrib_dir, "v{}".format(default_pg_version), "bin/postgres"
)
if os.getenv("REMOTE_ENV"):
# When testing against a remote server, we only need the client binary.
if not os.path.exists(os.path.join(pg_distrib_dir, "bin/psql")):
raise Exception('psql not found at "{}"'.format(pg_distrib_dir))
if not os.path.exists(psql_bin_path):
raise Exception('psql not found at "{}"'.format(psql_bin_path))
else:
if not os.path.exists(os.path.join(pg_distrib_dir, "bin/postgres")):
raise Exception('postgres not found at "{}"'.format(pg_distrib_dir))
if not os.path.exists(postgres_bin_path):
raise Exception('postgres not found at "{}"'.format(postgres_bin_path))
if os.getenv("REMOTE_ENV"):
# we are in remote env and do not have neon binaries locally
@@ -549,6 +556,7 @@ class NeonEnvBuilder:
self.env: Optional[NeonEnv] = None
self.remote_storage_prefix: Optional[str] = None
self.keep_remote_storage_contents: bool = True
self.pg_version = default_pg_version
def init(self) -> NeonEnv:
# Cannot create more than one environment from one builder
@@ -761,6 +769,7 @@ class NeonEnv:
self.broker = config.broker
self.remote_storage = config.remote_storage
self.remote_storage_users = config.remote_storage_users
self.pg_version = config.pg_version
# generate initial tenant ID here instead of letting 'neon init' generate it,
# so that we don't need to dig it out of the config file afterwards.
@@ -1195,7 +1204,6 @@ class AbstractNeonCli(abc.ABC):
env_vars = os.environ.copy()
env_vars["NEON_REPO_DIR"] = str(self.env.repo_dir)
env_vars["POSTGRES_DISTRIB_DIR"] = str(pg_distrib_dir)
env_vars["DEFAULT_PG_VERSION"] = str(pg_version)
if self.env.rust_log_override is not None:
env_vars["RUST_LOG"] = self.env.rust_log_override
for (extra_env_key, extra_env_value) in (extra_env_vars or {}).items():
@@ -1263,7 +1271,7 @@ class NeonCli(AbstractNeonCli):
"--timeline-id",
str(timeline_id),
"--pg-version",
pg_version,
self.env.pg_version,
]
)
else:
@@ -1276,7 +1284,7 @@ class NeonCli(AbstractNeonCli):
"--timeline-id",
str(timeline_id),
"--pg-version",
pg_version,
self.env.pg_version,
]
+ sum(list(map(lambda kv: (["-c", kv[0] + ":" + kv[1]]), conf.items())), [])
)
@@ -1302,7 +1310,9 @@ class NeonCli(AbstractNeonCli):
return res
def create_timeline(
self, new_branch_name: str, tenant_id: Optional[TenantId] = None
self,
new_branch_name: str,
tenant_id: Optional[TenantId] = None,
) -> TimelineId:
cmd = [
"timeline",
@@ -1312,7 +1322,7 @@ class NeonCli(AbstractNeonCli):
"--tenant-id",
str(tenant_id or self.env.initial_tenant),
"--pg-version",
pg_version,
self.env.pg_version,
]
res = self.raw_cli(cmd)
@@ -1326,7 +1336,11 @@ class NeonCli(AbstractNeonCli):
return TimelineId(str(created_timeline_id))
def create_root_branch(self, branch_name: str, tenant_id: Optional[TenantId] = None):
def create_root_branch(
self,
branch_name: str,
tenant_id: Optional[TenantId] = None,
):
cmd = [
"timeline",
"create",
@@ -1335,7 +1349,7 @@ class NeonCli(AbstractNeonCli):
"--tenant-id",
str(tenant_id or self.env.initial_tenant),
"--pg-version",
pg_version,
self.env.pg_version,
]
res = self.raw_cli(cmd)
@@ -1405,7 +1419,9 @@ class NeonCli(AbstractNeonCli):
return timelines_cli
def init(
self, config_toml: str, initial_timeline_id: Optional[TimelineId] = None
self,
config_toml: str,
initial_timeline_id: Optional[TimelineId] = None,
) -> "subprocess.CompletedProcess[str]":
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
tmp.write(config_toml)
@@ -1415,7 +1431,7 @@ class NeonCli(AbstractNeonCli):
if initial_timeline_id:
cmd.extend(["--timeline-id", str(initial_timeline_id)])
cmd.extend(["--pg-version", pg_version])
cmd.extend(["--pg-version", self.env.pg_version])
append_pageserver_param_overrides(
params_to_update=cmd,
@@ -1443,7 +1459,10 @@ class NeonCli(AbstractNeonCli):
log.info(f"pageserver_enabled_features success: {res.stdout}")
return json.loads(res.stdout)
def pageserver_start(self, overrides=()) -> "subprocess.CompletedProcess[str]":
def pageserver_start(
self,
overrides=(),
) -> "subprocess.CompletedProcess[str]":
start_args = ["pageserver", "start", *overrides]
append_pageserver_param_overrides(
params_to_update=start_args,
@@ -1499,7 +1518,7 @@ class NeonCli(AbstractNeonCli):
"--branch-name",
branch_name,
"--pg-version",
pg_version,
self.env.pg_version,
]
if lsn is not None:
args.extend(["--lsn", str(lsn)])
@@ -1525,7 +1544,7 @@ class NeonCli(AbstractNeonCli):
"--tenant-id",
str(tenant_id or self.env.initial_tenant),
"--pg-version",
pg_version,
self.env.pg_version,
]
if lsn is not None:
args.append(f"--lsn={lsn}")
@@ -1655,11 +1674,13 @@ def append_pageserver_param_overrides(
class PgBin:
"""A helper class for executing postgres binaries"""
def __init__(self, log_dir: Path):
def __init__(self, log_dir: Path, pg_version: str):
self.log_dir = log_dir
self.pg_bin_path = os.path.join(str(pg_distrib_dir), "bin")
self.pg_version = pg_version
self.pg_bin_path = os.path.join(str(pg_distrib_dir), "v{}".format(pg_version), "bin")
self.pg_lib_dir = os.path.join(str(pg_distrib_dir), "v{}".format(pg_version), "lib")
self.env = os.environ.copy()
self.env["LD_LIBRARY_PATH"] = os.path.join(str(pg_distrib_dir), "lib")
self.env["LD_LIBRARY_PATH"] = self.pg_lib_dir
def _fixpath(self, command: List[str]):
if "/" not in command[0]:
@@ -1714,8 +1735,8 @@ class PgBin:
@pytest.fixture(scope="function")
def pg_bin(test_output_dir: Path) -> PgBin:
return PgBin(test_output_dir)
def pg_bin(test_output_dir: Path, pg_version: str) -> PgBin:
return PgBin(test_output_dir, pg_version)
class VanillaPostgres(PgProtocol):
@@ -1762,12 +1783,19 @@ class VanillaPostgres(PgProtocol):
self.stop()
@pytest.fixture(scope="session")
def pg_version() -> str:
return default_pg_version
@pytest.fixture(scope="function")
def vanilla_pg(
test_output_dir: Path, port_distributor: PortDistributor
test_output_dir: Path,
port_distributor: PortDistributor,
pg_version: str,
) -> Iterator[VanillaPostgres]:
pgdatadir = test_output_dir / "pgdata-vanilla"
pg_bin = PgBin(test_output_dir)
pg_bin = PgBin(test_output_dir, pg_version)
port = port_distributor.get_port()
with VanillaPostgres(pgdatadir, pg_bin, port) as vanilla_pg:
yield vanilla_pg
@@ -1803,8 +1831,8 @@ class RemotePostgres(PgProtocol):
@pytest.fixture(scope="function")
def remote_pg(test_output_dir: Path) -> Iterator[RemotePostgres]:
pg_bin = PgBin(test_output_dir)
def remote_pg(test_output_dir: Path, pg_version: str) -> Iterator[RemotePostgres]:
pg_bin = PgBin(test_output_dir, pg_version)
connstr = os.getenv("BENCHMARK_CONNSTR")
if connstr is None:
@@ -2533,7 +2561,11 @@ def list_files_to_compare(pgdata_dir: Path):
# pg is the existing and running compute node, that we want to compare with a basebackup
def check_restored_datadir_content(test_output_dir: Path, env: NeonEnv, pg: Postgres):
def check_restored_datadir_content(
test_output_dir: Path,
env: NeonEnv,
pg: Postgres,
):
# Get the timeline ID. We need it for the 'basebackup' command
timeline = TimelineId(pg.safe_psql("SHOW neon.timeline_id")[0][0])
@@ -2544,7 +2576,7 @@ def check_restored_datadir_content(test_output_dir: Path, env: NeonEnv, pg: Post
restored_dir_path = env.repo_dir / f"{pg.node_name}_restored_datadir"
restored_dir_path.mkdir(exist_ok=True)
pg_bin = PgBin(test_output_dir)
pg_bin = PgBin(test_output_dir, env.pg_version)
psql_path = os.path.join(pg_bin.pg_bin_path, "psql")
cmd = rf"""
@@ -2557,7 +2589,7 @@ def check_restored_datadir_content(test_output_dir: Path, env: NeonEnv, pg: Post
# Set LD_LIBRARY_PATH in the env properly, otherwise we may use the wrong libpq.
# PgBin sets it automatically, but here we need to pipe psql output to the tar command.
psql_env = {"LD_LIBRARY_PATH": os.path.join(str(pg_distrib_dir), "lib")}
psql_env = {"LD_LIBRARY_PATH": pg_bin.pg_lib_dir}
result = subprocess.run(cmd, env=psql_env, capture_output=True, text=True, shell=True)
# Print captured stdout/stderr if basebackup cmd failed.

5
test_runner/regress/test_import.py
View File

@@ -14,7 +14,6 @@ from fixtures.neon_fixtures import (
PgBin,
Postgres,
pg_distrib_dir,
pg_version,
wait_for_last_record_lsn,
wait_for_upload,
)
@@ -98,7 +97,7 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build
"--wal-tarfile",
wal,
"--pg-version",
pg_version,
env.pg_version,
]
)
@@ -252,7 +251,7 @@ def _import(
"--base-tarfile",
os.path.join(tar_output_file),
"--pg-version",
pg_version,
env.pg_version,
]
)

26
test_runner/regress/test_pg_regress.py
View File

@@ -5,13 +5,7 @@ import os
from pathlib import Path
import pytest
from fixtures.neon_fixtures import (
NeonEnv,
base_dir,
check_restored_datadir_content,
pg_distrib_dir,
pg_version,
)
from fixtures.neon_fixtures import NeonEnv, base_dir, check_restored_datadir_content, pg_distrib_dir
# Run the main PostgreSQL regression tests, in src/test/regress.
@@ -32,9 +26,9 @@ def test_pg_regress(neon_simple_env: NeonEnv, test_output_dir: Path, pg_bin, cap
(runpath / "testtablespace").mkdir(parents=True)
# Compute all the file locations that pg_regress will need.
build_path = os.path.join(pg_distrib_dir, "../build/v{}/src/test/regress").format(pg_version)
src_path = os.path.join(base_dir, "vendor/postgres-v{}/src/test/regress").format(pg_version)
bindir = os.path.join(pg_distrib_dir, "bin")
build_path = os.path.join(pg_distrib_dir, "build/v{}/src/test/regress").format(env.pg_version)
src_path = os.path.join(base_dir, "vendor/postgres-v{}/src/test/regress").format(env.pg_version)
bindir = os.path.join(pg_distrib_dir, "v{}".format(env.pg_version), "bin")
schedule = os.path.join(src_path, "parallel_schedule")
pg_regress = os.path.join(build_path, "pg_regress")
@@ -86,9 +80,11 @@ def test_isolation(neon_simple_env: NeonEnv, test_output_dir: Path, pg_bin, caps
(runpath / "testtablespace").mkdir(parents=True)
# Compute all the file locations that pg_isolation_regress will need.
build_path = os.path.join(pg_distrib_dir, "../build/v{}/src/test/isolation".format(pg_version))
src_path = os.path.join(base_dir, "vendor/postgres-v{}/src/test/isolation".format(pg_version))
bindir = os.path.join(pg_distrib_dir, "bin")
build_path = os.path.join(pg_distrib_dir, "build/v{}/src/test/isolation".format(env.pg_version))
src_path = os.path.join(
base_dir, "vendor/postgres-v{}/src/test/isolation".format(env.pg_version)
)
bindir = os.path.join(pg_distrib_dir, "v{}".format(env.pg_version), "bin")
schedule = os.path.join(src_path, "isolation_schedule")
pg_isolation_regress = os.path.join(build_path, "pg_isolation_regress")
@@ -130,9 +126,9 @@ def test_sql_regress(neon_simple_env: NeonEnv, test_output_dir: Path, pg_bin, ca
# Compute all the file locations that pg_regress will need.
# This test runs neon specific tests
build_path = os.path.join(pg_distrib_dir, "../build/v{}/src/test/regress").format(pg_version)
build_path = os.path.join(pg_distrib_dir, "build/v{}/src/test/regress").format(env.pg_version)
src_path = os.path.join(base_dir, "test_runner/sql_regress")
bindir = os.path.join(pg_distrib_dir, "bin")
bindir = os.path.join(pg_distrib_dir, "v{}".format(env.pg_version), "bin")
schedule = os.path.join(src_path, "parallel_schedule")
pg_regress = os.path.join(build_path, "pg_regress")

3
test_runner/regress/test_wal_acceptor.py
View File

@@ -29,7 +29,6 @@ from fixtures.neon_fixtures import (
SafekeeperPort,
available_remote_storages,
neon_binpath,
pg_version,
wait_for_last_record_lsn,
wait_for_upload,
)
@@ -705,7 +704,7 @@ def test_sync_safekeepers(
"begin_lsn": int(begin_lsn),
"epoch_start_lsn": int(epoch_start_lsn),
"truncate_lsn": int(epoch_start_lsn),
"pg_version": int(pg_version) * 10000,
"pg_version": int(env.pg_version) * 10000,
},
)
lsn = Lsn(res["inserted_wal"]["end_lsn"])

4
test_runner/regress/test_wal_restore.py
View File

@@ -26,11 +26,11 @@ def test_wal_restore(
env.neon_cli.pageserver_stop()
port = port_distributor.get_port()
data_dir = test_output_dir / "pgsql.restored"
with VanillaPostgres(data_dir, PgBin(test_output_dir), port) as restored:
with VanillaPostgres(data_dir, PgBin(test_output_dir, env.pg_version), port) as restored:
pg_bin.run_capture(
[
os.path.join(base_dir, "libs/utils/scripts/restore_from_wal.sh"),
os.path.join(pg_distrib_dir, "bin"),
os.path.join(pg_distrib_dir, "v{}".format(env.pg_version), "bin"),
str(test_output_dir / "repo" / "safekeepers" / "sk1" / str(tenant_id) / "*"),
str(data_dir),
str(port),