rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-15 12:10:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

Increase inmem SMGR size for walredo process to 100 pagees (#10937)

Browse Source 
## Problem

We see `Inmem storage overflow` in page server logs:

https://neondb.slack.com/archives/C033RQ5SPDH/p1740157873114339

walked process is using inseam SMGR with storage size limited by 64
pages with warning watermark 32 (based ion the assumption that
XLR_MAX_BLOCK_ID is 32, so WAL record can not access more than 32
pages).

Actually it is not true. We can update up to 3 forks for each block
(including update of FSM and VM forks).

## Summary of changes

This PR increases inseam SMGR size for walled process to 100 pages and
print stack trace in case of overflow.

---------

Co-authored-by: Konstantin Knizhnik <knizhnik@neon.tech>
This commit is contained in:
Konstantin Knizhnik
2025-02-27 16:31:05 +02:00
committed by GitHub
parent a283edaccf
commit e58f264a05
3 changed files with 14 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pageserver/src/walredo/process.rs
View File

@@ -136,7 +136,9 @@ impl WalRedoProcess {
Ok(0) => break Ok(()), // eof
Ok(num_bytes) => {
let output = String::from_utf8_lossy(&buf[..num_bytes]);
error!(%output, "received output");
if !output.contains("LOG:") {
error!(%output, "received output");
}
}
Err(e) => {
break Err(e);

16
pgxn/neon_walredo/inmem_smgr.c
View File

@@ -32,8 +32,8 @@
#include "inmem_smgr.h"
/* Size of the in-memory smgr */
#define MAX_PAGES 64
/* Size of the in-memory smgr: XLR_MAX_BLOCK_ID is 32, but we can update up to 3 forks for each block */
#define MAX_PAGES 100
/* If more than WARN_PAGES are used, print a warning in the log */
#define WARN_PAGES 32
@@ -285,12 +285,12 @@ inmem_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
* WARN_PAGES, print a warning so that we get alerted and get to
* investigate why we're accessing so many buffers.
*/
elog(used_pages >= WARN_PAGES ? WARNING : DEBUG1,
"inmem_write() called for %u/%u/%u.%u blk %u: used_pages %u",
RelFileInfoFmt(InfoFromSMgrRel(reln)),
forknum,
blocknum,
used_pages);
if (used_pages >= WARN_PAGES)
ereport(WARNING, (errmsg("inmem_write() called for %u/%u/%u.%u blk %u: used_pages %u",
RelFileInfoFmt(InfoFromSMgrRel(reln)),
forknum,
blocknum,
used_pages), errbacktrace()));
if (used_pages == MAX_PAGES)
elog(ERROR, "Inmem storage overflow");

5
pgxn/neon_walredo/walredoproc.c
View File

@@ -142,7 +142,7 @@ static BufferTag target_redo_tag;
static XLogReaderState *reader_state;
#define TRACE LOG
#define TRACE DEBUG1
#ifdef HAVE_LIBSECCOMP
@@ -194,6 +194,7 @@ static PgSeccompRule allowed_syscalls[] =
* is stored in MyProcPid anyway.
*/
PG_SCMP_ALLOW(getpid),
PG_SCMP_ALLOW(futex), /* needed for errbacktrace */
/* Enable those for a proper shutdown. */
#if 0
@@ -253,7 +254,7 @@ WalRedoMain(int argc, char *argv[])
* which is super strange but that's not something we can solve
* for here. ¯\_(-_-)_/¯
*/
SetConfigOption("log_min_messages", "FATAL", PGC_SUSET, PGC_S_OVERRIDE);
SetConfigOption("log_min_messages", "WARNING", PGC_SUSET, PGC_S_OVERRIDE);
SetConfigOption("client_min_messages", "ERROR", PGC_SUSET,
PGC_S_OVERRIDE);