Add a 5 minute timeout to python tests.

The CI times out after 10 minutes of no output. It's annoying if a
test hangs and is killed by the CI timeout, because you don't get
information about which test was running. Try to avoid that, by adding
a slightly smaller timeout in pytest itself. You can override it on a
per-test basis if needed, but let's try to keep our tests shorter than
that.

For the Postgres regression tests, use a longer 30 minute timeout.
They're not really a single test, but many tests wrapped in a single
pytest test. It's OK for them to run longer in aggregate, each
Postgres test is still fairly short.

poetry.lock

@@ -1094,6 +1094,17 @@ python-versions = "*"
 [package.dependencies]
 pytest = ">=3.2.5"
 
+[[package]]
+name = "pytest-timeout"
+version = "2.1.0"
+description = "pytest plugin to abort hanging tests"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
+[package.dependencies]
+pytest = ">=5.0.0"
+
 [[package]]
 name = "pytest-xdist"
 version = "2.5.0"
@@ -1387,7 +1398,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "d2fcba2af0a32cde3a1d0c8cfdfe5fb26531599b0c8c376bf16e200a74b55553"
+content-hash = "4ee85b435461dec70b406bf7170302fe54e9e247bdf628a9cb6b5fb9eb9afd82"
 
 [metadata.files]
 aiopg = [
@@ -1889,6 +1900,10 @@ pytest-lazy-fixture = [
     {file = "pytest-lazy-fixture-0.6.3.tar.gz", hash = "sha256:0e7d0c7f74ba33e6e80905e9bfd81f9d15ef9a790de97993e34213deb5ad10ac"},
     {file = "pytest_lazy_fixture-0.6.3-py3-none-any.whl", hash = "sha256:e0b379f38299ff27a653f03eaa69b08a6fd4484e46fd1c9907d984b9f9daeda6"},
 ]
+pytest-timeout = [
+    {file = "pytest-timeout-2.1.0.tar.gz", hash = "sha256:c07ca07404c612f8abbe22294b23c368e2e5104b521c1790195561f37e1ac3d9"},
+    {file = "pytest_timeout-2.1.0-py3-none-any.whl", hash = "sha256:f6f50101443ce70ad325ceb4473c4255e9d74e3c7cd0ef827309dfa4c0d975c6"},
+]
 pytest-xdist = [
     {file = "pytest-xdist-2.5.0.tar.gz", hash = "sha256:4580deca3ff04ddb2ac53eba39d76cb5dd5edeac050cb6fbc768b0dd712b4edf"},
     {file = "pytest_xdist-2.5.0-py3-none-any.whl", hash = "sha256:6fe5c74fec98906deb8f2d2b616b5c782022744978e7bd4695d39c8f42d0ce65"},

pyproject.toml

@@ -24,6 +24,7 @@ moto = {version = "^3.0.0", extras = ["server"]}
 backoff = "^1.11.1"
 pytest-lazy-fixture = "^0.6.3"
 prometheus-client = "^0.14.1"
+pytest-timeout = "^2.1.0"
 
 [tool.poetry.dev-dependencies]
 yapf = "==0.31.0"

pytest.ini

@@ -9,3 +9,4 @@ minversion = 6.0
 log_format = %(asctime)s.%(msecs)-3d %(levelname)s [%(filename)s:%(lineno)d] %(message)s
 log_date_format = %Y-%m-%d %H:%M:%S
 log_cli = true
+timeout = 300

test_runner/batch_pg_regress/test_isolation.py

@@ -1,9 +1,12 @@
 import os
-
+import pytest
 from fixtures.utils import mkdir_if_needed
 from fixtures.zenith_fixtures import ZenithEnv, base_dir, pg_distrib_dir
 
 
+# The isolation tests run for a long time, especially in debug mode,
+# so use a larger-than-default timeout.
+@pytest.mark.timeout(1800)
 def test_isolation(zenith_simple_env: ZenithEnv, test_output_dir, pg_bin, capsys):
     env = zenith_simple_env
 

test_runner/batch_pg_regress/test_pg_regress.py

@@ -1,9 +1,12 @@
 import os
-
+import pytest
 from fixtures.utils import mkdir_if_needed
 from fixtures.zenith_fixtures import ZenithEnv, check_restored_datadir_content, base_dir, pg_distrib_dir
 
 
+# The pg_regress tests run for a long time, especially in debug mode,
+# so use a larger-than-default timeout.
+@pytest.mark.timeout(1800)
 def test_pg_regress(zenith_simple_env: ZenithEnv, test_output_dir: str, pg_bin, capsys):
     env = zenith_simple_env
 

test_runner/performance/test_startup.py

@@ -1,9 +1,11 @@
+import pytest
 from contextlib import closing
-
 from fixtures.zenith_fixtures import ZenithEnvBuilder
 from fixtures.benchmark_fixture import ZenithBenchmarker
 
 
+# This test sometimes runs for longer than the global 5 minute timeout.
+@pytest.mark.timeout(600)
 def test_startup(zenith_env_builder: ZenithEnvBuilder, zenbenchmark: ZenithBenchmarker):
     zenith_env_builder.num_safekeepers = 3
     env = zenith_env_builder.init_start()