Add slow seqscan perf test (#1283)

test_runner/fixtures/compare_fixtures.py

@@ -25,6 +25,10 @@ class PgCompare(ABC):
     def pg_bin(self) -> PgBin:
         pass
 
+    @property
+    def zenbenchmark(self) -> ZenithBenchmarker:
+        pass
+
     @abstractmethod
     def flush(self) -> None:
         pass
@@ -56,7 +60,7 @@ class ZenithCompare(PgCompare):
                  pg_bin: PgBin,
                  branch_name):
         self.env = zenith_simple_env
-        self.zenbenchmark = zenbenchmark
+        self._zenbenchmark = zenbenchmark
         self._pg_bin = pg_bin
 
         # We only use one branch and one timeline
@@ -73,6 +77,10 @@ class ZenithCompare(PgCompare):
     def pg(self):
         return self._pg
 
+    @property
+    def zenbenchmark(self):
+        return self._zenbenchmark
+
     @property
     def pg_bin(self):
         return self._pg_bin
@@ -106,7 +114,7 @@ class VanillaCompare(PgCompare):
     """PgCompare interface for vanilla postgres."""
     def __init__(self, zenbenchmark, vanilla_pg: VanillaPostgres):
         self._pg = vanilla_pg
-        self.zenbenchmark = zenbenchmark
+        self._zenbenchmark = zenbenchmark
         vanilla_pg.configure(['shared_buffers=1MB'])
         vanilla_pg.start()
 
@@ -118,6 +126,10 @@ class VanillaCompare(PgCompare):
     def pg(self):
         return self._pg
 
+    @property
+    def zenbenchmark(self):
+        return self._zenbenchmark
+
     @property
     def pg_bin(self):
         return self._pg.pg_bin

test_runner/fixtures/slow.py

@@ -0,0 +1,26 @@
+import pytest
+"""
+This plugin allows tests to be marked as slow using pytest.mark.slow. By default slow
+tests are excluded. They need to be specifically requested with the --runslow flag in
+order to run.
+
+Copied from here: https://docs.pytest.org/en/latest/example/simple.html
+"""
+
+
+def pytest_addoption(parser):
+    parser.addoption("--runslow", action="store_true", default=False, help="run slow tests")
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "slow: mark test as slow to run")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--runslow"):
+        # --runslow given in cli: do not skip slow tests
+        return
+    skip_slow = pytest.mark.skip(reason="need --runslow option to run")
+    for item in items:
+        if "slow" in item.keywords:
+            item.add_marker(skip_slow)