diff --git a/test_runner/regress/test_flaky.py b/test_runner/regress/test_flaky.py
new file mode 100644
index 0000000000..4a0f01cd14
--- /dev/null
+++ b/test_runner/regress/test_flaky.py
@@ -0,0 +1,11 @@
+"""Test for detecting new flaky tests"""
+
+import random
+
+
+def test_flaky1():
+    assert random.random() > 0.05
+
+
+def no_test_flaky2():
+    assert random.random() > 0.05
diff --git a/test_runner/regress/test_setup.py b/test_runner/regress/test_setup.py
index fd0f5705c8..ce7bd2cba1 100644
--- a/test_runner/regress/test_setup.py
+++ b/test_runner/regress/test_setup.py
@@ -11,6 +11,9 @@ if TYPE_CHECKING:
 # Test that pageserver and safekeeper can restart quickly.
 # This is a regression test, see https://github.com/neondatabase/neon/issues/2247
 def test_fixture_restart(neon_env_builder: NeonEnvBuilder):
+    import random
+
+    assert random.random() > 0.05
     env = neon_env_builder.init_start()
 
     for _ in range(3):
@@ -20,3 +23,9 @@ def test_fixture_restart(neon_env_builder: NeonEnvBuilder):
     for _ in range(3):
         env.safekeepers[0].stop()
         env.safekeepers[0].start()
+
+
+def test_flaky3():
+    import random
+
+    assert random.random() > 0.05