From dc7004ec10cb37f0b13dd4627de4b1fd44ab6a8a Mon Sep 17 00:00:00 2001
From: Vlad Lazar <vlad@neon.tech>
Date: Thu, 31 Oct 2024 13:49:48 +0100
Subject: [PATCH] wal_decoder: add rate limited log for unexpecte rm ids

Problem

We would like to handle the tightening of unexpected WAL record
types, but don't know if this happens in the field.

Summary of Changes

Add a rate-limited WAL log on such events. The rate limiting is to
play it safe and guard against spewing in prod.
---
 Cargo.lock                      |  1 +
 libs/wal_decoder/Cargo.toml     |  1 +
 libs/wal_decoder/src/decoder.rs | 17 ++++++++++++++++-
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/Cargo.lock b/Cargo.lock
index c5af247e8b..59d2bc01da 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6962,6 +6962,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "bytes",
+ "once_cell",
  "pageserver_api",
  "postgres_ffi",
  "serde",
diff --git a/libs/wal_decoder/Cargo.toml b/libs/wal_decoder/Cargo.toml
index 3f80f8fcdb..a76dac2ed1 100644
--- a/libs/wal_decoder/Cargo.toml
+++ b/libs/wal_decoder/Cargo.toml
@@ -10,6 +10,7 @@ testing = []
 [dependencies]
 anyhow.workspace = true
 bytes.workspace = true
+once_cell.workspace = true
 pageserver_api.workspace = true
 postgres_ffi.workspace = true
 serde.workspace = true
diff --git a/libs/wal_decoder/src/decoder.rs b/libs/wal_decoder/src/decoder.rs
index 780fce3d69..06144d45b8 100644
--- a/libs/wal_decoder/src/decoder.rs
+++ b/libs/wal_decoder/src/decoder.rs
@@ -169,9 +169,24 @@ impl MetadataRecord {
             }
             pg_constants::RM_STANDBY_ID => Self::decode_standby_record(&mut buf, decoded),
             pg_constants::RM_REPLORIGIN_ID => Self::decode_replorigin_record(&mut buf, decoded),
-            _unexpected => {
+            unexpected => {
                 // TODO: consider failing here instead of blindly doing something without
                 // understanding the protocol
+                use once_cell::sync::Lazy;
+                use std::sync::Mutex;
+                use std::time::Duration;
+                use utils::rate_limit::RateLimit;
+
+                static LOGGED: Lazy<Mutex<RateLimit>> =
+                    Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
+                let mut rate_limit = LOGGED.try_lock().unwrap();
+                rate_limit.call(|| {
+                    tracing::warn!(
+                        "Unexpected resource manager id in PG WAL record at LSN {}: {}",
+                        lsn,
+                        unexpected
+                    );
+                });
                 Ok(None)
             }
         }