mirror of
https://github.com/neondatabase/neon.git
synced 2026-07-07 22:20:36 +00:00
fast import: added a test that restores into a running postgres by connstring
This commit is contained in:
@@ -50,8 +50,9 @@ class FastImport(AbstractNeonCli):
|
||||
|
||||
def run(
|
||||
self,
|
||||
pg_port: int,
|
||||
pg_port: int | None = None,
|
||||
source_connection_string: str | None = None,
|
||||
restore_connection_string: str | None = None,
|
||||
s3prefix: str | None = None,
|
||||
interactive: bool = False,
|
||||
) -> subprocess.CompletedProcess[str]:
|
||||
@@ -60,11 +61,14 @@ class FastImport(AbstractNeonCli):
|
||||
args = [
|
||||
f"--pg-bin-dir={self.pg_bin}",
|
||||
f"--pg-lib-dir={self.pg_lib}",
|
||||
f"--pg-port={pg_port}",
|
||||
f"--working-directory={self.workdir}",
|
||||
]
|
||||
if pg_port is not None:
|
||||
args.append(f"--pg-port={pg_port}")
|
||||
if source_connection_string is not None:
|
||||
args.append(f"--source-connection-string={source_connection_string}")
|
||||
if restore_connection_string is not None:
|
||||
args.append(f"--restore-connection-string={restore_connection_string}")
|
||||
if s3prefix is not None:
|
||||
args.append(f"--s3-prefix={s3prefix}")
|
||||
if interactive:
|
||||
|
||||
@@ -2,6 +2,7 @@ import json
|
||||
import re
|
||||
import time
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg2
|
||||
import psycopg2.errors
|
||||
@@ -14,6 +15,7 @@ from fixtures.pageserver.http import (
|
||||
ImportPgdataIdemptencyKey,
|
||||
PageserverApiException,
|
||||
)
|
||||
from fixtures.pg_version import PgVersion
|
||||
from fixtures.port_distributor import PortDistributor
|
||||
from fixtures.remote_storage import RemoteStorageKind
|
||||
from pytest_httpserver import HTTPServer
|
||||
@@ -94,13 +96,15 @@ def test_pgdata_import_smoke(
|
||||
while True:
|
||||
relblock_size = vanilla_pg.safe_psql_scalar("select pg_relation_size('t')")
|
||||
log.info(
|
||||
f"relblock size: {relblock_size/8192} pages (target: {target_relblock_size//8192}) pages"
|
||||
f"relblock size: {relblock_size / 8192} pages (target: {target_relblock_size // 8192}) pages"
|
||||
)
|
||||
if relblock_size >= target_relblock_size:
|
||||
break
|
||||
addrows = int((target_relblock_size - relblock_size) // 8192)
|
||||
assert addrows >= 1, "forward progress"
|
||||
vanilla_pg.safe_psql(f"insert into t select generate_series({nrows+1}, {nrows + addrows})")
|
||||
vanilla_pg.safe_psql(
|
||||
f"insert into t select generate_series({nrows + 1}, {nrows + addrows})"
|
||||
)
|
||||
nrows += addrows
|
||||
expect_nrows = nrows
|
||||
expect_sum = (
|
||||
@@ -336,6 +340,36 @@ def test_fast_import_binary(
|
||||
new_pgdata_vanilla_pg.stop()
|
||||
|
||||
|
||||
def test_fast_import_restore_to_connstring(
|
||||
test_output_dir,
|
||||
vanilla_pg: VanillaPostgres,
|
||||
port_distributor: PortDistributor,
|
||||
fast_import: FastImport,
|
||||
pg_distrib_dir: Path,
|
||||
pg_version: PgVersion,
|
||||
):
|
||||
vanilla_pg.start()
|
||||
vanilla_pg.safe_psql("CREATE TABLE foo (a int); INSERT INTO foo SELECT generate_series(1, 10);")
|
||||
|
||||
pgdatadir = test_output_dir / "restore-pgdata"
|
||||
pg_bin = PgBin(test_output_dir, pg_distrib_dir, pg_version)
|
||||
port = port_distributor.get_port()
|
||||
with VanillaPostgres(pgdatadir, pg_bin, port) as restore_vanilla_pg:
|
||||
restore_vanilla_pg.configure(["shared_preload_libraries='neon_rmgr'"])
|
||||
restore_vanilla_pg.start()
|
||||
|
||||
fast_import.run(
|
||||
source_connection_string=vanilla_pg.connstr(),
|
||||
restore_connection_string=restore_vanilla_pg.connstr(),
|
||||
)
|
||||
vanilla_pg.stop()
|
||||
|
||||
# database name and user are hardcoded in fast_import binary, and they are different from normal vanilla postgres
|
||||
res = restore_vanilla_pg.safe_psql("SELECT count(*) FROM foo;")
|
||||
log.info(f"Result: {res}")
|
||||
assert res[0][0] == 10
|
||||
|
||||
|
||||
# TODO: Maybe test with pageserver?
|
||||
# 1. run whole neon env
|
||||
# 2. create timeline with some s3 path???
|
||||
|
||||
Reference in New Issue
Block a user