From 42fa2a44f2d2a6a4bb5e62bc70c2a69894289c74 Mon Sep 17 00:00:00 2001
From: Arseny Sher <sher-ars@yandex.ru>
Date: Tue, 28 Mar 2023 11:55:52 +0400
Subject: [PATCH] Use pytest-repeat to reproduce failure on CI.

---
 .../actions/run-python-test-set/action.yml    |  4 ++--
 .github/workflows/build_and_test.yml          |  1 +
 poetry.lock                                   | 19 +++++++++++++++++--
 pyproject.toml                                |  1 +
 test_runner/fixtures/neon_fixtures.py         |  2 +-
 test_runner/fixtures/utils.py                 |  2 +-
 test_runner/regress/test_compatibility.py     |  1 +
 7 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml
index 29b04a3478..5e48d655bb 100644
--- a/.github/actions/run-python-test-set/action.yml
+++ b/.github/actions/run-python-test-set/action.yml
@@ -19,7 +19,7 @@ inputs:
   run_in_parallel:
     description: 'Whether to run tests in parallel'
     required: false
-    default: 'true'
+    default: 'false'
   save_perf_report:
     description: 'Whether to upload the performance report, if true PERF_TEST_RESULT_CONNSTR env variable should be set'
     required: false
@@ -171,7 +171,7 @@ runs:
           --junitxml=$TEST_OUTPUT/junit.xml \
           --alluredir=$TEST_OUTPUT/allure/results \
           --tb=short \
-          --verbose \
+          --verbose -k "test_forward or test_create_snapsh" -x \
           -rA $TEST_SELECTION $EXTRA_PARAMS
 
         if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index e056cf0fcf..c9d3947f12 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - release
+      - tmp-repro
   pull_request:
 
 defaults:
diff --git a/poetry.lock b/poetry.lock
index 011d5d7817..d13d3ccd97 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand.
 
 [[package]]
 name = "aiohttp"
@@ -1932,6 +1932,21 @@ pytest = [
     {version = ">=6.2.4", markers = "python_version >= \"3.10\""},
 ]
 
+[[package]]
+name = "pytest-repeat"
+version = "0.9.1"
+description = "pytest plugin for repeating tests"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "pytest-repeat-0.9.1.tar.gz", hash = "sha256:5cd3289745ab3156d43eb9c8e7f7d00a926f3ae5c9cf425bec649b2fe15bad5b"},
+    {file = "pytest_repeat-0.9.1-py2.py3-none-any.whl", hash = "sha256:4474a7d9e9137f6d8cc8ae297f8c4168d33c56dd740aa78cfffe562557e6b96e"},
+]
+
+[package.dependencies]
+pytest = ">=3.6"
+
 [[package]]
 name = "pytest-timeout"
 version = "2.1.0"
@@ -2597,4 +2612,4 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "2515a9320c2960076012fbc036fb33c4f6a23515c8d143785931dc18c6722d91"
+content-hash = "ce00f0cf8735e850b1cfb29d8ddf8f66fab2395c589fe7e16becc774edcc7845"
diff --git a/pyproject.toml b/pyproject.toml
index f21c12b2e3..e47bb6fc4d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,6 +34,7 @@ types-psutil = "^5.9.5.4"
 types-toml = "^0.10.8"
 pytest-httpserver = "^1.0.6"
 aiohttp = "3.7.4"
+pytest-repeat = "^0.9.1"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.1.0"
diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py
index 8b228ad804..d8c50f0960 100644
--- a/test_runner/fixtures/neon_fixtures.py
+++ b/test_runner/fixtures/neon_fixtures.py
@@ -77,7 +77,7 @@ DEFAULT_BRANCH_NAME: str = "main"
 DEFAULT_PG_VERSION_DEFAULT: str = "14"
 
 BASE_PORT: int = 15000
-WORKER_PORT_NUM: int = 1000
+WORKER_PORT_NUM: int = 10000
 
 
 def pytest_configure(config: Config):
diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py
index ce03658e8f..0e70193f36 100644
--- a/test_runner/fixtures/utils.py
+++ b/test_runner/fixtures/utils.py
@@ -149,7 +149,7 @@ def get_scale_for_db(size_mb: int) -> int:
 
 
 ATTACHMENT_NAME_REGEX: re.Pattern = re.compile(  # type: ignore[type-arg]
-    r"regression\.diffs|.+\.(?:log|stderr|stdout|filediff|metrics|html)"
+    r"config|postgresql\.conf|regression\.diffs|.+\.(?:log|stderr|stdout|filediff|metrics|html)"
 )
 
 
diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py
index 66625dd6f8..d0e3a81926 100644
--- a/test_runner/regress/test_compatibility.py
+++ b/test_runner/regress/test_compatibility.py
@@ -136,6 +136,7 @@ def test_backward_compatibility(
 
 @pytest.mark.xdist_group("compatibility")
 @pytest.mark.order(after="test_create_snapshot")
+@pytest.mark.repeat(1000)
 def test_forward_compatibility(
     test_output_dir: Path,
     port_distributor: PortDistributor,