Add test of clog (pg_xact) truncation

Makefile

@@ -51,6 +51,8 @@ postgres: postgres-configure
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 install
 	+@echo "Compiling contrib/zenith"
 	$(MAKE) -C tmp_install/build/contrib/zenith install
+	+@echo "Compiling contrib/zenith_test_utils"
+	$(MAKE) -C tmp_install/build/contrib/zenith_test_utils install
 
 postgres-clean:
 	$(MAKE) -C tmp_install/build MAKELEVEL=0 clean

pageserver/src/restore_local_repo.rs

@@ -708,14 +708,15 @@ fn save_xact_record(
 fn save_clog_truncate_record(
     checkpoint: &mut CheckPoint,
     _timeline: &dyn Timeline,
-    _lsn: Lsn,
+    lsn: Lsn,
     xlrec: &XlClogTruncate,
 ) -> Result<()> {
-    trace!(
-        "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {}",
+    info!(
+        "RM_CLOG_ID truncate pageno {} oldestXid {} oldestXidDB {} lsn {}",
         xlrec.pageno,
         xlrec.oldest_xid,
-        xlrec.oldest_xid_db
+        xlrec.oldest_xid_db,
+        lsn
     );
 
     checkpoint.oldestXid = xlrec.oldest_xid;

test_runner/batch_others/test_clog_truncate.py

@@ -0,0 +1,72 @@
+import time
+import os
+
+from contextlib import closing
+
+from fixtures.zenith_fixtures import PostgresFactory, ZenithPageserver
+
+pytest_plugins = ("fixtures.zenith_fixtures")
+
+
+#
+# Test compute node start after clog truncation
+#
+def test_clog_truncate(zenith_cli, pageserver: ZenithPageserver, postgres: PostgresFactory, pg_bin):
+    # Create a branch for us
+    zenith_cli.run(["branch", "test_clog_truncate", "empty"])
+
+    # set agressive autovacuum to make sure that truncation will happen
+    config = [
+        'autovacuum_max_workers=10', 'autovacuum_vacuum_threshold=0',
+        'autovacuum_vacuum_insert_threshold=0', 'autovacuum_vacuum_cost_delay=0',
+        'autovacuum_vacuum_cost_limit=10000', 'autovacuum_naptime =1s',
+        'autovacuum_freeze_max_age=100000'
+    ]
+
+    pg = postgres.create_start('test_clog_truncate', config_lines=config)
+    print('postgres is running on test_clog_truncate branch')
+
+    # Install extension containing function needed for test
+    pg.safe_psql('CREATE EXTENSION zenith_test_utils')
+
+    # Consume many xids to advance clog
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('select test_consume_xids(1000*1000*10);')
+            print('xids consumed')
+
+            # call a checkpoint to trigger TruncateSubtrans
+            cur.execute('CHECKPOINT;')
+
+            # ensure WAL flush
+            cur.execute('select txid_current()')
+            print(cur.fetchone())
+
+    # wait for autovacuum to truncate the pg_xact
+    # XXX Is it worth to add a timeout here?
+    pg_xact_0000_path = os.path.join(pg.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path = " + pg_xact_0000_path)
+
+    while os.path.isfile(pg_xact_0000_path):
+        print("file exists. wait for truncation. " "pg_xact_0000_path = " + pg_xact_0000_path)
+        time.sleep(5)
+
+    # checkpoint to advance latest lsn
+    with closing(pg.connect()) as conn:
+        with conn.cursor() as cur:
+            cur.execute('CHECKPOINT;')
+            cur.execute('select pg_current_wal_insert_lsn()')
+            lsn_after_truncation = cur.fetchone()[0]
+
+    # create new branch after clog truncation and start a compute node on it
+    print('create branch at lsn_after_truncation ' + lsn_after_truncation)
+    zenith_cli.run(
+        ["branch", "test_clog_truncate_new", "test_clog_truncate@" + lsn_after_truncation])
+
+    pg2 = postgres.create_start('test_clog_truncate_new')
+    print('postgres is running on test_clog_truncate_new branch')
+
+    # check that new node doesn't contain truncated segment
+    pg_xact_0000_path_new = os.path.join(pg2.pg_xact_dir_path(), '0000')
+    print("pg_xact_0000_path_new = " + pg_xact_0000_path_new)
+    assert os.path.isfile(pg_xact_0000_path_new) is False

test_runner/fixtures/zenith_fixtures.py

@@ -284,6 +284,11 @@ class Postgres(PgProtocol):
 
         return self
 
+    def pg_xact_dir_path(self) -> str:
+        """ Path to pg_xact dir """
+        path = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'pg_xact'
+        return os.path.join(self.repo_dir, path)
+
     def config_file_path(self) -> str:
         """ Path to postgresql.conf """
         filename = pathlib.Path('pgdatadirs') / 'tenants' / self.tenant_id / self.branch / 'postgresql.conf'