wal_decoder: add rate limited log for unexpecte rm ids

Problem

We would like to handle the tightening of unexpected WAL record
types, but don't know if this happens in the field.

Summary of Changes

Add a rate-limited WAL log on such events. The rate limiting is to
play it safe and guard against spewing in prod.

Cargo.lock

@@ -6962,6 +6962,7 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "bytes",
+ "once_cell",
  "pageserver_api",
  "postgres_ffi",
  "serde",

libs/wal_decoder/Cargo.toml

@@ -10,6 +10,7 @@ testing = []
 [dependencies]
 anyhow.workspace = true
 bytes.workspace = true
+once_cell.workspace = true
 pageserver_api.workspace = true
 postgres_ffi.workspace = true
 serde.workspace = true

libs/wal_decoder/src/decoder.rs

@@ -169,9 +169,24 @@ impl MetadataRecord {
             }
             pg_constants::RM_STANDBY_ID => Self::decode_standby_record(&mut buf, decoded),
             pg_constants::RM_REPLORIGIN_ID => Self::decode_replorigin_record(&mut buf, decoded),
-            _unexpected => {
+            unexpected => {
                 // TODO: consider failing here instead of blindly doing something without
                 // understanding the protocol
+                use once_cell::sync::Lazy;
+                use std::sync::Mutex;
+                use std::time::Duration;
+                use utils::rate_limit::RateLimit;
+
+                static LOGGED: Lazy<Mutex<RateLimit>> =
+                    Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(10))));
+                let mut rate_limit = LOGGED.try_lock().unwrap();
+                rate_limit.call(|| {
+                    tracing::warn!(
+                        "Unexpected resource manager id in PG WAL record at LSN {}: {}",
+                        lsn,
+                        unexpected
+                    );
+                });
                 Ok(None)
             }
         }