Add slow seqscan perf test (#1283)

test_runner/conftest.py

@@ -1 +1,6 @@
-pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+pytest_plugins = (
+    "fixtures.zenith_fixtures",
+    "fixtures.benchmark_fixture",
+    "fixtures.compare_fixtures",
+    "fixtures.slow",
+)

test_runner/fixtures/compare_fixtures.py

@@ -25,6 +25,10 @@ class PgCompare(ABC):
     def pg_bin(self) -> PgBin:
         pass
 
+    @property
+    def zenbenchmark(self) -> ZenithBenchmarker:
+        pass
+
     @abstractmethod
     def flush(self) -> None:
         pass
@@ -56,7 +60,7 @@ class ZenithCompare(PgCompare):
                  pg_bin: PgBin,
                  branch_name):
         self.env = zenith_simple_env
-        self.zenbenchmark = zenbenchmark
+        self._zenbenchmark = zenbenchmark
         self._pg_bin = pg_bin
 
         # We only use one branch and one timeline
@@ -73,6 +77,10 @@ class ZenithCompare(PgCompare):
     def pg(self):
         return self._pg
 
+    @property
+    def zenbenchmark(self):
+        return self._zenbenchmark
+
     @property
     def pg_bin(self):
         return self._pg_bin
@@ -106,7 +114,7 @@ class VanillaCompare(PgCompare):
     """PgCompare interface for vanilla postgres."""
     def __init__(self, zenbenchmark, vanilla_pg: VanillaPostgres):
         self._pg = vanilla_pg
-        self.zenbenchmark = zenbenchmark
+        self._zenbenchmark = zenbenchmark
         vanilla_pg.configure(['shared_buffers=1MB'])
         vanilla_pg.start()
 
@@ -118,6 +126,10 @@ class VanillaCompare(PgCompare):
     def pg(self):
         return self._pg
 
+    @property
+    def zenbenchmark(self):
+        return self._zenbenchmark
+
     @property
     def pg_bin(self):
         return self._pg.pg_bin

test_runner/fixtures/slow.py

@@ -0,0 +1,26 @@
+import pytest
+"""
+This plugin allows tests to be marked as slow using pytest.mark.slow. By default slow
+tests are excluded. They need to be specifically requested with the --runslow flag in
+order to run.
+
+Copied from here: https://docs.pytest.org/en/latest/example/simple.html
+"""
+
+
+def pytest_addoption(parser):
+    parser.addoption("--runslow", action="store_true", default=False, help="run slow tests")
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "slow: mark test as slow to run")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)

test_runner/performance/test_small_seqscans.py

@@ -4,10 +4,12 @@
 # cache, so the seqscans go to the page server. But small enough that it fits
 # into memory in the page server.
 from contextlib import closing
+from dataclasses import dataclass
 from fixtures.zenith_fixtures import ZenithEnv
 from fixtures.log_helper import log
 from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
 from fixtures.compare_fixtures import PgCompare
+import pytest
 
 pytest_plugins = (
     "fixtures.zenith_fixtures",
@@ -16,13 +18,17 @@ pytest_plugins = (
 )
 
 
-def test_small_seqscans(zenith_with_baseline: PgCompare):
+@pytest.mark.parametrize('rows', [
+    pytest.param(100000),
+    pytest.param(1000000, marks=pytest.mark.slow),
+])
+def test_small_seqscans(zenith_with_baseline: PgCompare, rows: int):
     env = zenith_with_baseline
 
     with closing(env.pg.connect()) as conn:
         with conn.cursor() as cur:
             cur.execute('create table t (i integer);')
-            cur.execute('insert into t values (generate_series(1,100000));')
+            cur.execute(f'insert into t values (generate_series(1,{rows}));')
 
             # Verify that the table is larger than shared_buffers
             cur.execute('''
@@ -30,8 +36,11 @@ def test_small_seqscans(zenith_with_baseline: PgCompare):
             from pg_settings where name = 'shared_buffers'
             ''')
             row = cur.fetchone()
-            log.info(f"shared_buffers is {row[0]}, table size {row[1]}")
-            assert int(row[0]) < int(row[1])
+            shared_buffers = row[0]
+            table_size = row[1]
+            log.info(f"shared_buffers is {shared_buffers}, table size {table_size}")
+            assert int(shared_buffers) < int(table_size)
+            env.zenbenchmark.record("table_size", table_size, 'bytes', MetricReport.TEST_PARAM)
 
             with env.record_duration('run'):
                 for i in range(1000):