mirror of
https://github.com/neondatabase/neon.git
synced 2026-05-18 21:50:37 +00:00
Merge branch 'main' into bojan-get-page-tests
This commit is contained in:
Bojan Serafimov
@@ -2,29 +2,113 @@ from contextlib import closing
|
||||
from fixtures.zenith_fixtures import PgBin, VanillaPostgres, ZenithEnv
|
||||
from fixtures.compare_fixtures import PgCompare, VanillaCompare, ZenithCompare
|
||||
|
||||
from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
|
||||
from fixtures.benchmark_fixture import PgBenchRunResult, MetricReport, ZenithBenchmarker
|
||||
from fixtures.log_helper import log
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
import calendar
|
||||
import os
|
||||
import timeit
|
||||
|
||||
|
||||
def utc_now_timestamp() -> int:
|
||||
return calendar.timegm(datetime.utcnow().utctimetuple())
|
||||
|
||||
|
||||
def init_pgbench(env: PgCompare, cmdline):
|
||||
# calculate timestamps and durations separately
|
||||
# timestamp is intended to be used for linking to grafana and logs
|
||||
# duration is actually a metric and uses float instead of int for timestamp
|
||||
init_start_timestamp = utc_now_timestamp()
|
||||
t0 = timeit.default_timer()
|
||||
with env.record_pageserver_writes('init.pageserver_writes'):
|
||||
env.pg_bin.run_capture(cmdline)
|
||||
env.flush()
|
||||
init_duration = timeit.default_timer() - t0
|
||||
init_end_timestamp = utc_now_timestamp()
|
||||
|
||||
env.zenbenchmark.record("init.duration",
|
||||
init_duration,
|
||||
unit="s",
|
||||
report=MetricReport.LOWER_IS_BETTER)
|
||||
env.zenbenchmark.record("init.start_timestamp",
|
||||
init_start_timestamp,
|
||||
'',
|
||||
MetricReport.TEST_PARAM)
|
||||
env.zenbenchmark.record("init.end_timestamp", init_end_timestamp, '', MetricReport.TEST_PARAM)
|
||||
|
||||
|
||||
def run_pgbench(env: PgCompare, prefix: str, cmdline):
|
||||
with env.record_pageserver_writes(f'{prefix}.pageserver_writes'):
|
||||
run_start_timestamp = utc_now_timestamp()
|
||||
t0 = timeit.default_timer()
|
||||
out = env.pg_bin.run_capture(cmdline, )
|
||||
run_duration = timeit.default_timer() - t0
|
||||
run_end_timestamp = utc_now_timestamp()
|
||||
env.flush()
|
||||
|
||||
stdout = Path(f"{out}.stdout").read_text()
|
||||
|
||||
res = PgBenchRunResult.parse_from_stdout(
|
||||
stdout=stdout,
|
||||
run_duration=run_duration,
|
||||
run_start_timestamp=run_start_timestamp,
|
||||
run_end_timestamp=run_end_timestamp,
|
||||
)
|
||||
env.zenbenchmark.record_pg_bench_result(prefix, res)
|
||||
|
||||
|
||||
#
|
||||
# Run a very short pgbench test.
|
||||
# Initialize a pgbench database, and run pgbench against it.
|
||||
#
|
||||
# Collects three metrics:
|
||||
# This makes runs two different pgbench workloads against the same
|
||||
# initialized database, and 'duration' is the time of each run. So
|
||||
# the total runtime is 2 * duration, plus time needed to initialize
|
||||
# the test database.
|
||||
#
|
||||
# 1. Time to initialize the pgbench database (pgbench -s5 -i)
|
||||
# 2. Time to run 5000 pgbench transactions
|
||||
# 3. Disk space used
|
||||
#
|
||||
def test_pgbench(zenith_with_baseline: PgCompare):
|
||||
env = zenith_with_baseline
|
||||
# Currently, the # of connections is hardcoded at 4
|
||||
def run_test_pgbench(env: PgCompare, scale: int, duration: int):
|
||||
|
||||
with env.record_pageserver_writes('pageserver_writes'):
|
||||
with env.record_duration('init'):
|
||||
env.pg_bin.run_capture(['pgbench', '-s5', '-i', env.pg.connstr()])
|
||||
env.flush()
|
||||
# Record the scale and initialize
|
||||
env.zenbenchmark.record("scale", scale, '', MetricReport.TEST_PARAM)
|
||||
init_pgbench(env, ['pgbench', f'-s{scale}', '-i', env.pg.connstr()])
|
||||
|
||||
with env.record_duration('5000_xacts'):
|
||||
env.pg_bin.run_capture(['pgbench', '-c1', '-t5000', env.pg.connstr()])
|
||||
env.flush()
|
||||
# Run simple-update workload
|
||||
run_pgbench(env,
|
||||
"simple-update",
|
||||
['pgbench', '-n', '-c4', f'-T{duration}', '-P2', '-Mprepared', env.pg.connstr()])
|
||||
|
||||
# Run SELECT workload
|
||||
run_pgbench(env,
|
||||
"select-only",
|
||||
['pgbench', '-S', '-c4', f'-T{duration}', '-P2', '-Mprepared', env.pg.connstr()])
|
||||
|
||||
env.report_size()
|
||||
|
||||
|
||||
def get_durations_matrix():
|
||||
durations = os.getenv("TEST_PG_BENCH_DURATIONS_MATRIX", default="45")
|
||||
return list(map(int, durations.split(",")))
|
||||
|
||||
|
||||
def get_scales_matrix():
|
||||
scales = os.getenv("TEST_PG_BENCH_SCALES_MATRIX", default="10")
|
||||
return list(map(int, scales.split(",")))
|
||||
|
||||
|
||||
# Run the pgbench tests against vanilla Postgres and zenith
|
||||
@pytest.mark.parametrize("scale", get_scales_matrix())
|
||||
@pytest.mark.parametrize("duration", get_durations_matrix())
|
||||
def test_pgbench(zenith_with_baseline: PgCompare, scale: int, duration: int):
|
||||
run_test_pgbench(zenith_with_baseline, scale, duration)
|
||||
|
||||
|
||||
# Run the pgbench tests against an existing Postgres cluster
|
||||
@pytest.mark.parametrize("scale", get_scales_matrix())
|
||||
@pytest.mark.parametrize("duration", get_durations_matrix())
|
||||
@pytest.mark.remote_cluster
|
||||
def test_pgbench_remote(remote_compare: PgCompare, scale: int, duration: int):
|
||||
run_test_pgbench(remote_compare, scale, duration)
|
||||
|
||||
@@ -1,124 +0,0 @@
|
||||
import dataclasses
|
||||
import os
|
||||
import subprocess
|
||||
from typing import List
|
||||
from fixtures.benchmark_fixture import PgBenchRunResult, ZenithBenchmarker
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
import calendar
|
||||
import timeit
|
||||
import os
|
||||
|
||||
|
||||
def utc_now_timestamp() -> int:
|
||||
return calendar.timegm(datetime.utcnow().utctimetuple())
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PgBenchRunner:
|
||||
connstr: str
|
||||
scale: int
|
||||
transactions: int
|
||||
pgbench_bin_path: str = "pgbench"
|
||||
|
||||
def invoke(self, args: List[str]) -> 'subprocess.CompletedProcess[str]':
|
||||
res = subprocess.run([self.pgbench_bin_path, *args], text=True, capture_output=True)
|
||||
|
||||
if res.returncode != 0:
|
||||
raise RuntimeError(f"pgbench failed. stdout: {res.stdout} stderr: {res.stderr}")
|
||||
return res
|
||||
|
||||
def init(self, vacuum: bool = True) -> 'subprocess.CompletedProcess[str]':
|
||||
args = []
|
||||
if not vacuum:
|
||||
args.append("--no-vacuum")
|
||||
args.extend([f"--scale={self.scale}", "--initialize", self.connstr])
|
||||
return self.invoke(args)
|
||||
|
||||
def run(self, jobs: int = 1, clients: int = 1):
|
||||
return self.invoke([
|
||||
f"--transactions={self.transactions}",
|
||||
f"--jobs={jobs}",
|
||||
f"--client={clients}",
|
||||
"--progress=2", # print progress every two seconds
|
||||
self.connstr,
|
||||
])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def connstr():
|
||||
res = os.getenv("BENCHMARK_CONNSTR")
|
||||
if res is None:
|
||||
raise ValueError("no connstr provided, use BENCHMARK_CONNSTR environment variable")
|
||||
return res
|
||||
|
||||
|
||||
def get_transactions_matrix():
|
||||
transactions = os.getenv("TEST_PG_BENCH_TRANSACTIONS_MATRIX")
|
||||
if transactions is None:
|
||||
return [10**4, 10**5]
|
||||
return list(map(int, transactions.split(",")))
|
||||
|
||||
|
||||
def get_scales_matrix():
|
||||
scales = os.getenv("TEST_PG_BENCH_SCALES_MATRIX")
|
||||
if scales is None:
|
||||
return [10, 20]
|
||||
return list(map(int, scales.split(",")))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale", get_scales_matrix())
|
||||
@pytest.mark.parametrize("transactions", get_transactions_matrix())
|
||||
@pytest.mark.remote_cluster
|
||||
def test_pg_bench_remote_cluster(zenbenchmark: ZenithBenchmarker,
|
||||
connstr: str,
|
||||
scale: int,
|
||||
transactions: int):
|
||||
"""
|
||||
The best way is to run same pack of tests both, for local zenith
|
||||
and against staging, but currently local tests heavily depend on
|
||||
things available only locally e.g. zenith binaries, pageserver api, etc.
|
||||
Also separate test allows to run pgbench workload against vanilla postgres
|
||||
or other systems that support postgres protocol.
|
||||
|
||||
Also now this is more of a liveness test because it stresses pageserver internals,
|
||||
so we clearly see what goes wrong in more "real" environment.
|
||||
"""
|
||||
pg_bin = os.getenv("PG_BIN")
|
||||
if pg_bin is not None:
|
||||
pgbench_bin_path = os.path.join(pg_bin, "pgbench")
|
||||
else:
|
||||
pgbench_bin_path = "pgbench"
|
||||
|
||||
runner = PgBenchRunner(
|
||||
connstr=connstr,
|
||||
scale=scale,
|
||||
transactions=transactions,
|
||||
pgbench_bin_path=pgbench_bin_path,
|
||||
)
|
||||
# calculate timestamps and durations separately
|
||||
# timestamp is intended to be used for linking to grafana and logs
|
||||
# duration is actually a metric and uses float instead of int for timestamp
|
||||
init_start_timestamp = utc_now_timestamp()
|
||||
t0 = timeit.default_timer()
|
||||
runner.init()
|
||||
init_duration = timeit.default_timer() - t0
|
||||
init_end_timestamp = utc_now_timestamp()
|
||||
|
||||
run_start_timestamp = utc_now_timestamp()
|
||||
t0 = timeit.default_timer()
|
||||
out = runner.run() # TODO handle failures
|
||||
run_duration = timeit.default_timer() - t0
|
||||
run_end_timestamp = utc_now_timestamp()
|
||||
|
||||
res = PgBenchRunResult.parse_from_output(
|
||||
out=out,
|
||||
init_duration=init_duration,
|
||||
init_start_timestamp=init_start_timestamp,
|
||||
init_end_timestamp=init_end_timestamp,
|
||||
run_duration=run_duration,
|
||||
run_start_timestamp=run_start_timestamp,
|
||||
run_end_timestamp=run_end_timestamp,
|
||||
)
|
||||
|
||||
zenbenchmark.record_pg_bench_result(res)
|
||||
Reference in New Issue
Block a user