Add flaky tests to test how testing for flaky tests works

test_runner/regress/test_flaky.py

@@ -0,0 +1,11 @@
+"""Test for detecting new flaky tests"""
+
+import random
+
+
+def test_flaky1():
+    assert random.random() > 0.05
+
+
+def no_test_flaky2():
+    assert random.random() > 0.05

test_runner/regress/test_setup.py

@@ -11,6 +11,9 @@ if TYPE_CHECKING:
 # Test that pageserver and safekeeper can restart quickly.
 # This is a regression test, see https://github.com/neondatabase/neon/issues/2247
 def test_fixture_restart(neon_env_builder: NeonEnvBuilder):
+    import random
+
+    assert random.random() > 0.05
     env = neon_env_builder.init_start()
 
     for _ in range(3):
@@ -20,3 +23,9 @@ def test_fixture_restart(neon_env_builder: NeonEnvBuilder):
     for _ in range(3):
         env.safekeepers[0].stop()
         env.safekeepers[0].start()
+
+
+def test_flaky3():
+    import random
+
+    assert random.random() > 0.05