fast import: added a test that restores into a running postgres by connstring

This commit is contained in:
Gleb Novikov
2025-01-16 18:05:21 +00:00
parent 047b986f7f
commit 0dda2ad7ec
2 changed files with 42 additions and 4 deletions

View File

@@ -50,8 +50,9 @@ class FastImport(AbstractNeonCli):
def run(
self,
pg_port: int,
pg_port: int | None = None,
source_connection_string: str | None = None,
restore_connection_string: str | None = None,
s3prefix: str | None = None,
interactive: bool = False,
) -> subprocess.CompletedProcess[str]:
@@ -60,11 +61,14 @@ class FastImport(AbstractNeonCli):
args = [
f"--pg-bin-dir={self.pg_bin}",
f"--pg-lib-dir={self.pg_lib}",
f"--pg-port={pg_port}",
f"--working-directory={self.workdir}",
]
if pg_port is not None:
args.append(f"--pg-port={pg_port}")
if source_connection_string is not None:
args.append(f"--source-connection-string={source_connection_string}")
if restore_connection_string is not None:
args.append(f"--restore-connection-string={restore_connection_string}")
if s3prefix is not None:
args.append(f"--s3-prefix={s3prefix}")
if interactive:

View File

@@ -2,6 +2,7 @@ import json
import re
import time
from enum import Enum
from pathlib import Path
import psycopg2
import psycopg2.errors
@@ -14,6 +15,7 @@ from fixtures.pageserver.http import (
ImportPgdataIdemptencyKey,
PageserverApiException,
)
from fixtures.pg_version import PgVersion
from fixtures.port_distributor import PortDistributor
from fixtures.remote_storage import RemoteStorageKind
from pytest_httpserver import HTTPServer
@@ -94,13 +96,15 @@ def test_pgdata_import_smoke(
while True:
relblock_size = vanilla_pg.safe_psql_scalar("select pg_relation_size('t')")
log.info(
f"relblock size: {relblock_size/8192} pages (target: {target_relblock_size//8192}) pages"
f"relblock size: {relblock_size / 8192} pages (target: {target_relblock_size // 8192}) pages"
)
if relblock_size >= target_relblock_size:
break
addrows = int((target_relblock_size - relblock_size) // 8192)
assert addrows >= 1, "forward progress"
vanilla_pg.safe_psql(f"insert into t select generate_series({nrows+1}, {nrows + addrows})")
vanilla_pg.safe_psql(
f"insert into t select generate_series({nrows + 1}, {nrows + addrows})"
)
nrows += addrows
expect_nrows = nrows
expect_sum = (
@@ -336,6 +340,36 @@ def test_fast_import_binary(
new_pgdata_vanilla_pg.stop()
def test_fast_import_restore_to_connstring(
test_output_dir,
vanilla_pg: VanillaPostgres,
port_distributor: PortDistributor,
fast_import: FastImport,
pg_distrib_dir: Path,
pg_version: PgVersion,
):
vanilla_pg.start()
vanilla_pg.safe_psql("CREATE TABLE foo (a int); INSERT INTO foo SELECT generate_series(1, 10);")
pgdatadir = test_output_dir / "restore-pgdata"
pg_bin = PgBin(test_output_dir, pg_distrib_dir, pg_version)
port = port_distributor.get_port()
with VanillaPostgres(pgdatadir, pg_bin, port) as restore_vanilla_pg:
restore_vanilla_pg.configure(["shared_preload_libraries='neon_rmgr'"])
restore_vanilla_pg.start()
fast_import.run(
source_connection_string=vanilla_pg.connstr(),
restore_connection_string=restore_vanilla_pg.connstr(),
)
vanilla_pg.stop()
# database name and user are hardcoded in fast_import binary, and they are different from normal vanilla postgres
res = restore_vanilla_pg.safe_psql("SELECT count(*) FROM foo;")
log.info(f"Result: {res}")
assert res[0][0] == 10
# TODO: Maybe test with pageserver?
# 1. run whole neon env
# 2. create timeline with some s3 path???