From 32aa1fc68151a7393801447e2f33688d78b07ea1 Mon Sep 17 00:00:00 2001 From: Sasha Krassovsky Date: Mon, 12 Aug 2024 21:54:42 -0700 Subject: [PATCH] Add on-demand WAL download to slot funcs (#8705) ## Problem Currently we can have an issue where if someone does `pg_logical_slot_advance`, it could fail because it doesn't have the WAL locally. ## Summary of changes Adds on-demand WAL download and a test to these slot funcs. Before adding these, the test fails with ``` requested WAL segment pg_wal/000000010000000000000001 has already been removed ``` After the changes, the test passes Relies on: - https://github.com/neondatabase/postgres/pull/466 - https://github.com/neondatabase/postgres/pull/467 - https://github.com/neondatabase/postgres/pull/468 --- pgxn/neon/neon.c | 1 + test_runner/regress/test_logical_replication.py | 15 +++++++++++++++ vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/revisions.json | 6 +++--- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/pgxn/neon/neon.c b/pgxn/neon/neon.c index d107cdc1c2..9cdbf4a126 100644 --- a/pgxn/neon/neon.c +++ b/pgxn/neon/neon.c @@ -599,6 +599,7 @@ _PG_init(void) pg_init_walproposer(); WalSender_Custom_XLogReaderRoutines = NeonOnDemandXLogReaderRoutines; LogicalFuncs_Custom_XLogReaderRoutines = NeonOnDemandXLogReaderRoutines; + SlotFuncs_Custom_XLogReaderRoutines = NeonOnDemandXLogReaderRoutines; InitLogicalReplicationMonitor(); diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index 5a5d369a11..0d18aa43b7 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -255,6 +255,21 @@ FROM generate_series(1, 16384) AS seq; -- Inserts enough rows to exceed 16MB of cur.execute( "SELECT * FROM pg_logical_slot_peek_binary_changes('slotty_mcslotface', NULL, NULL, 'include-xids', '0')" ) + cur.execute( + """ +INSERT INTO wal_generator (data) +SELECT repeat('A', 1024) -- Generates a kilobyte of data per row +FROM generate_series(1, 16384) AS seq; -- Inserts enough rows to exceed 16MB of data +""" + ) + + endpoint.stop_and_destroy() + endpoint = env.endpoints.create_start("init") + with endpoint.connect().cursor() as cur: + log.info("advance slot") + cur.execute( + "SELECT * from pg_replication_slot_advance('slotty_mcslotface', pg_current_wal_lsn())" + ) # Tests that walsender correctly blocks until WAL is downloaded from safekeepers diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index ae07734e0f..a48faca1d9 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit ae07734e0ff72759ab425fc8f625d4c1ecb15a50 +Subproject commit a48faca1d9aef59649dd1bf34bc1b6303fa3489e diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 47c8d462d1..39c51c33b3 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 47c8d462d169367c8979ce628a523be2d94b46be +Subproject commit 39c51c33b383239c78b86afe561679f980e44842 diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 6434b1499b..5ea106b258 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 6434b1499b11ed97dccea5618a055034b83b8e2f +Subproject commit 5ea106b2583285849784e774b39d62eb2615bd5d diff --git a/vendor/revisions.json b/vendor/revisions.json index ab8b3b3c4f..f983407268 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,14 +1,14 @@ { "v16": [ "16.3", - "6434b1499b11ed97dccea5618a055034b83b8e2f" + "5ea106b2583285849784e774b39d62eb2615bd5d" ], "v15": [ "15.7", - "47c8d462d169367c8979ce628a523be2d94b46be" + "39c51c33b383239c78b86afe561679f980e44842" ], "v14": [ "14.12", - "ae07734e0ff72759ab425fc8f625d4c1ecb15a50" + "a48faca1d9aef59649dd1bf34bc1b6303fa3489e" ] }