From d0dc65da124d3f84e2f64ac5e3927b0a299c9eab Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Fri, 9 May 2025 18:12:49 +0800 Subject: [PATCH] fix(pageserver): give up gc-compaction if one key has too long history (#11869) ## Problem The limitation we imposed last week https://github.com/neondatabase/neon/pull/11709 is not enough to protect excessive memory usage. ## Summary of changes If a single key accumulated too much history, give up compaction. In the future, we can make the `generate_key_retention` function take a stream of keys instead of first accumulating them in memory, thus easily support such long key history cases. Signed-off-by: Alex Chi Z --- pageserver/src/tenant/timeline/compaction.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 07cd274a41..6b155268d6 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -3606,6 +3606,13 @@ impl Timeline { last_key = Some(key); } accumulated_values.push((key, lsn, val)); + + if accumulated_values.len() >= 65536 { + // Assume all of them are images, that would be 512MB of data in memory for a single key. + return Err(CompactionError::Other(anyhow!( + "too many values for a single key, giving up gc-compaction" + ))); + } } else { let last_key: &mut Key = last_key.as_mut().unwrap(); stat.on_unique_key_visited(); // TODO: adjust statistics for partial compaction