try to repro

.github/actions/run-python-test-set/action.yml

@@ -171,7 +171,7 @@ runs:
           --junitxml=$TEST_OUTPUT/junit.xml \
           --alluredir=$TEST_OUTPUT/allure/results \
           --tb=short \
-          --verbose \
+          --verbose -k "test_forward or test_create_snapsh" -x \
           -rA $TEST_SELECTION $EXTRA_PARAMS
 
         if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then

.github/workflows/build_and_test.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - release
+      - tmp-repro
   pull_request:
 
 defaults:

libs/utils/src/lsn.rs

@@ -6,6 +6,7 @@ use std::ops::{Add, AddAssign};
 use std::path::Path;
 use std::str::FromStr;
 use std::sync::atomic::{AtomicU64, Ordering};
+use tracing::info;
 
 use crate::seqwait::MonotonicCounter;
 
@@ -239,6 +240,7 @@ impl MonotonicCounter<Lsn> for RecordLsn {
         let new_prev = self.last;
         self.last = lsn;
         self.prev = new_prev;
+        info!("advanced record lsn to {}/{}", self.last, self.prev);
     }
     fn cnt_value(&self) -> Lsn {
         self.last

pageserver/src/tenant/timeline.rs

@@ -1246,6 +1246,11 @@ impl Timeline {
 
                 state,
             };
+            info!(
+                "initialized lsrlsn to {}/{}",
+                disk_consistent_lsn,
+                metadata.prev_record_lsn().unwrap_or(Lsn(0))
+            );
             result.repartition_threshold = result.get_checkpoint_distance() / 10;
             result
                 .metrics

poetry.lock

@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.4.0 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.4.1 and should not be changed by hand.
 
 [[package]]
 name = "aiohttp"
@@ -1932,6 +1932,21 @@ pytest = [
     {version = ">=6.2.4", markers = "python_version >= \"3.10\""},
 ]
 
+[[package]]
+name = "pytest-repeat"
+version = "0.9.1"
+description = "pytest plugin for repeating tests"
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+files = [
+    {file = "pytest-repeat-0.9.1.tar.gz", hash = "sha256:5cd3289745ab3156d43eb9c8e7f7d00a926f3ae5c9cf425bec649b2fe15bad5b"},
+    {file = "pytest_repeat-0.9.1-py2.py3-none-any.whl", hash = "sha256:4474a7d9e9137f6d8cc8ae297f8c4168d33c56dd740aa78cfffe562557e6b96e"},
+]
+
+[package.dependencies]
+pytest = ">=3.6"
+
 [[package]]
 name = "pytest-timeout"
 version = "2.1.0"
@@ -2597,4 +2612,4 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.9"
-content-hash = "2515a9320c2960076012fbc036fb33c4f6a23515c8d143785931dc18c6722d91"
+content-hash = "ce00f0cf8735e850b1cfb29d8ddf8f66fab2395c589fe7e16becc774edcc7845"

pyproject.toml

@@ -34,6 +34,7 @@ types-psutil = "^5.9.5.4"
 types-toml = "^0.10.8"
 pytest-httpserver = "^1.0.6"
 aiohttp = "3.7.4"
+pytest-repeat = "^0.9.1"
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.1.0"

test_runner/fixtures/utils.py

@@ -149,7 +149,7 @@ def get_scale_for_db(size_mb: int) -> int:
 
 
 ATTACHMENT_NAME_REGEX: re.Pattern = re.compile(  # type: ignore[type-arg]
-    r"regression\.diffs|.+\.(?:log|stderr|stdout|filediff|metrics|html)"
+    r"config|postgresql\.conf|regression\.diffs|.+\.(?:log|stderr|stdout|filediff|metrics|html)"
 )
 
 

test_runner/regress/test_compatibility.py

@@ -136,6 +136,7 @@ def test_backward_compatibility(
 
 @pytest.mark.xdist_group("compatibility")
 @pytest.mark.order(after="test_create_snapshot")
+@pytest.mark.repeat(1000)
 def test_forward_compatibility(
     test_output_dir: Path,
     port_distributor: PortDistributor,