Use pytest plugin

poetry.lock

@@ -814,7 +814,7 @@ python-versions = "*"
 
 [[package]]
 name = "moto"
-version = "3.0.0"
+version = "3.0.3"
 description = "A library that allows your python tests to easily mock out the boto library"
 category = "main"
 optional = false
@@ -849,6 +849,7 @@ xmltodict = "*"
 [package.extras]
 all = ["PyYAML (>=5.1)", "python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)", "docker (>=2.5.1)", "graphql-core", "jsondiff (>=1.1.2)", "aws-xray-sdk (>=0.93,!=0.96)", "idna (>=2.5,<4)", "cfn-lint (>=0.4.0)", "sshpubkeys (>=3.1.0)", "setuptools"]
 apigateway = ["python-jose[cryptography] (>=3.1.0,<4.0.0)", "ecdsa (!=0.15)"]
+apigatewayv2 = ["PyYAML (>=5.1)"]
 appsync = ["graphql-core"]
 awslambda = ["docker (>=2.5.1)"]
 batch = ["docker (>=2.5.1)"]
@@ -1059,6 +1060,20 @@ python-versions = ">=3.6"
 py = "*"
 pytest = ">=3.10"
 
+[[package]]
+name = "pytest-skip-slow"
+version = "0.0.2"
+description = "A pytest plugin to skip `@pytest.mark.slow` tests by default. "
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+pytest = ">=6.2.0"
+
+[package.extras]
+test = ["tox"]
+
 [[package]]
 name = "pytest-xdist"
 version = "2.5.0"
@@ -1352,7 +1367,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "0fa6c9377fbc827240d18d8b7e3742def37e90fc3277fddf8525d82dabd13090"
+content-hash = "d59ea97fb78d13dcede2719734c55b7d6b9dcc7f86001a9228c8808ed6e58eb7"
 
 [metadata.files]
 aiopg = [
@@ -1666,8 +1681,8 @@ mccabe = [
     {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
 ]
 moto = [
-    {file = "moto-3.0.0-py2.py3-none-any.whl", hash = "sha256:762d33bbad3642c687f6495e69331318bef43f9aa662174397706ec3ad2a3578"},
-    {file = "moto-3.0.0.tar.gz", hash = "sha256:d6b00a2663290e7ebb06823d5ffcb124c8dc9bf526b878539ef7c4a377fd8255"},
+    {file = "moto-3.0.3-py2.py3-none-any.whl", hash = "sha256:445a574395b8a43a249ae0f932bf10c5cc677054198bfa1ff92e6fbd60e72c38"},
+    {file = "moto-3.0.3.tar.gz", hash = "sha256:fa3fbdc22c55d7e70b407e2f2639c48ac82b074f472b167609405c0c1e3a2ccb"},
 ]
 mypy = [
     {file = "mypy-0.910-cp35-cp35m-macosx_10_9_x86_64.whl", hash = "sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457"},
@@ -1842,6 +1857,10 @@ pytest-forked = [
     {file = "pytest-forked-1.4.0.tar.gz", hash = "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e"},
     {file = "pytest_forked-1.4.0-py3-none-any.whl", hash = "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8"},
 ]
+pytest-skip-slow = [
+    {file = "pytest-skip-slow-0.0.2.tar.gz", hash = "sha256:06b8353d8b1e2168c22b8c34172329b4f592eea648dac141099cc881fd466718"},
+    {file = "pytest_skip_slow-0.0.2-py3-none-any.whl", hash = "sha256:9029eb8c258e6c3c4b499848d864c9e9c48499d8b8bf5bb413044f59874cfd06"},
+]
 pytest-xdist = [
     {file = "pytest-xdist-2.5.0.tar.gz", hash = "sha256:4580deca3ff04ddb2ac53eba39d76cb5dd5edeac050cb6fbc768b0dd712b4edf"},
     {file = "pytest_xdist-2.5.0-py3-none-any.whl", hash = "sha256:6fe5c74fec98906deb8f2d2b616b5c782022744978e7bd4695d39c8f42d0ce65"},

pyproject.toml

@@ -21,6 +21,7 @@ types-psycopg2 = "^2.9.6"
 boto3 = "^1.20.40"
 boto3-stubs = "^1.20.40"
 moto = {version = "^3.0.0", extras = ["server"]}
+pytest-skip-slow = "^0.0.2"
 
 [tool.poetry.dev-dependencies]
 yapf = "==0.31.0"

test_runner/fixtures/compare_fixtures.py

@@ -25,6 +25,10 @@ class PgCompare(ABC):
     def pg_bin(self) -> PgBin:
         pass
 
+    @property
+    def zenbenchmark(self) -> ZenithBenchmarker:
+        pass
+
     @abstractmethod
     def flush(self) -> None:
         pass
@@ -56,7 +60,7 @@ class ZenithCompare(PgCompare):
                  pg_bin: PgBin,
                  branch_name):
         self.env = zenith_simple_env
-        self.zenbenchmark = zenbenchmark
+        self._zenbenchmark = zenbenchmark
         self._pg_bin = pg_bin
 
         # We only use one branch and one timeline
@@ -73,6 +77,10 @@ class ZenithCompare(PgCompare):
     def pg(self):
         return self._pg
 
+    @property
+    def zenbenchmark(self):
+        return self._zenbenchmark
+
     @property
     def pg_bin(self):
         return self._pg_bin
@@ -106,7 +114,7 @@ class VanillaCompare(PgCompare):
     """PgCompare interface for vanilla postgres."""
     def __init__(self, zenbenchmark, vanilla_pg: VanillaPostgres):
         self._pg = vanilla_pg
-        self.zenbenchmark = zenbenchmark
+        self._zenbenchmark = zenbenchmark
         vanilla_pg.configure(['shared_buffers=1MB'])
         vanilla_pg.start()
 
@@ -118,6 +126,10 @@ class VanillaCompare(PgCompare):
     def pg(self):
         return self._pg
 
+    @property
+    def zenbenchmark(self):
+        return self._zenbenchmark
+
     @property
     def pg_bin(self):
         return self._pg.pg_bin

test_runner/performance/test_small_seqscans.py

@@ -4,10 +4,12 @@
 # cache, so the seqscans go to the page server. But small enough that it fits
 # into memory in the page server.
 from contextlib import closing
+from dataclasses import dataclass
 from fixtures.zenith_fixtures import ZenithEnv
 from fixtures.log_helper import log
 from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
 from fixtures.compare_fixtures import PgCompare
+import pytest
 
 pytest_plugins = (
     "fixtures.zenith_fixtures",
@@ -16,13 +18,17 @@ pytest_plugins = (
 )
 
 
-def test_small_seqscans(zenith_with_baseline: PgCompare):
+@pytest.mark.parametrize('rows', [
+    pytest.param(100000),
+    pytest.param(1000000, marks=pytest.mark.slow),
+])
+def test_small_seqscans(zenith_with_baseline: PgCompare, rows: int):
     env = zenith_with_baseline
 
     with closing(env.pg.connect()) as conn:
         with conn.cursor() as cur:
             cur.execute('create table t (i integer);')
-            cur.execute('insert into t values (generate_series(1,100000));')
+            cur.execute(f'insert into t values (generate_series(1,{rows}));')
 
             # Verify that the table is larger than shared_buffers
             cur.execute('''
@@ -30,8 +36,11 @@ def test_small_seqscans(zenith_with_baseline: PgCompare):
             from pg_settings where name = 'shared_buffers'
             ''')
             row = cur.fetchone()
-            log.info(f"shared_buffers is {row[0]}, table size {row[1]}")
-            assert int(row[0]) < int(row[1])
+            shared_buffers = row[0]
+            table_size = row[1]
+            log.info(f"shared_buffers is {shared_buffers}, table size {table_size}")
+            assert int(shared_buffers) < int(table_size)
+            env.zenbenchmark.record("table_size", table_size, 'bytes', MetricReport.TEST_PARAM)
 
             with env.record_duration('run'):
                 for i in range(1000):