From 552fa2b9729fe323176dc93c73fecc531dc7cc53 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 3 Oct 2024 16:40:35 +0100 Subject: [PATCH] pageserver: tweak oversized key read path warning (#9221) ## Problem `Oversized vectored read [...]` logs are spewing in prod because we have a few keys that are unexpectedly large: * reldir/relblock - these are unbounded, so it's known technical debt * slru block - they can be a bit bigger than 128KiB due to storage format overhead ## Summary of changes * Bump threshold to 130KiB * Don't warn on oversized reldir and dbdir keys Closes https://github.com/neondatabase/neon/issues/8967 --- libs/pageserver_api/src/config.rs | 9 ++++++- libs/pageserver_api/src/key.rs | 10 +++++++ .../src/tenant/storage_layer/delta_layer.rs | 26 ++++++++++++++----- .../src/tenant/storage_layer/image_layer.rs | 26 ++++++++++++++----- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/libs/pageserver_api/src/config.rs b/libs/pageserver_api/src/config.rs index 95310fdbac..105c8a50d3 100644 --- a/libs/pageserver_api/src/config.rs +++ b/libs/pageserver_api/src/config.rs @@ -296,7 +296,14 @@ pub mod defaults { pub const DEFAULT_INGEST_BATCH_SIZE: u64 = 100; - pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 128 * 1024; // 128 KiB + /// Soft limit for the maximum size of a vectored read. + /// + /// This is determined by the largest NeonWalRecord that can exist (minus dbdir and reldir keys + /// which are bounded by the blob io limits only). As of this writing, that is a `NeonWalRecord::ClogSetCommitted` record, + /// with 32k xids. That's the max number of XIDS on a single CLOG page. The size of such a record + /// is `sizeof(Transactionid) * 32768 + (some fixed overhead from 'timestamp`, the Vec length and whatever extra serde serialization adds)`. + /// That is, slightly above 128 kB. + pub const DEFAULT_MAX_VECTORED_READ_BYTES: usize = 130 * 1024; // 130 KiB pub const DEFAULT_IMAGE_COMPRESSION: ImageCompressionAlgorithm = ImageCompressionAlgorithm::Zstd { level: Some(1) }; diff --git a/libs/pageserver_api/src/key.rs b/libs/pageserver_api/src/key.rs index 4a776709c9..b3fcaae62f 100644 --- a/libs/pageserver_api/src/key.rs +++ b/libs/pageserver_api/src/key.rs @@ -748,6 +748,16 @@ impl Key { self.field1 == 0x00 && self.field4 != 0 && self.field6 != 0xffffffff } + #[inline(always)] + pub fn is_rel_dir_key(&self) -> bool { + self.field1 == 0x00 + && self.field2 != 0 + && self.field3 != 0 + && self.field4 == 0 + && self.field5 == 0 + && self.field6 == 1 + } + /// Guaranteed to return `Ok()` if [`Self::is_rel_block_key`] returns `true` for `key`. #[inline(always)] pub fn to_rel_block(self) -> anyhow::Result<(RelTag, BlockNumber)> { diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 6f9eda85f5..2acad666b8 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -53,6 +53,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use futures::StreamExt; use itertools::Itertools; use pageserver_api::config::MaxVectoredReadBytes; +use pageserver_api::key::DBDIR_KEY; use pageserver_api::keyspace::KeySpace; use pageserver_api::models::ImageCompressionAlgorithm; use pageserver_api::shard::TenantShardId; @@ -963,14 +964,25 @@ impl DeltaLayerInner { .blobs_at .as_slice() .iter() - .map(|(_, blob_meta)| format!("{}@{}", blob_meta.key, blob_meta.lsn)) + .filter_map(|(_, blob_meta)| { + if blob_meta.key.is_rel_dir_key() || blob_meta.key == DBDIR_KEY { + // The size of values for these keys is unbounded and can + // grow very large in pathological cases. + None + } else { + Some(format!("{}@{}", blob_meta.key, blob_meta.lsn)) + } + }) .join(", "); - tracing::warn!( - "Oversized vectored read ({} > {}) for keys {}", - largest_read_size, - read_size_soft_max, - offenders - ); + + if !offenders.is_empty() { + tracing::warn!( + "Oversized vectored read ({} > {}) for keys {}", + largest_read_size, + read_size_soft_max, + offenders + ); + } } largest_read_size diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 3dcd7bc962..9b53fa9e18 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -49,6 +49,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use hex; use itertools::Itertools; use pageserver_api::config::MaxVectoredReadBytes; +use pageserver_api::key::DBDIR_KEY; use pageserver_api::keyspace::KeySpace; use pageserver_api::shard::{ShardIdentity, TenantShardId}; use rand::{distributions::Alphanumeric, Rng}; @@ -587,14 +588,25 @@ impl ImageLayerInner { .blobs_at .as_slice() .iter() - .map(|(_, blob_meta)| format!("{}@{}", blob_meta.key, blob_meta.lsn)) + .filter_map(|(_, blob_meta)| { + if blob_meta.key.is_rel_dir_key() || blob_meta.key == DBDIR_KEY { + // The size of values for these keys is unbounded and can + // grow very large in pathological cases. + None + } else { + Some(format!("{}@{}", blob_meta.key, blob_meta.lsn)) + } + }) .join(", "); - tracing::warn!( - "Oversized vectored read ({} > {}) for keys {}", - buf_size, - max_vectored_read_bytes, - offenders - ); + + if !offenders.is_empty() { + tracing::warn!( + "Oversized vectored read ({} > {}) for keys {}", + buf_size, + max_vectored_read_bytes, + offenders + ); + } } let buf = BytesMut::with_capacity(buf_size);