rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-07 05:22:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pageserver): yield image creation to L0 compactions across timelines (#10877)

Browse Source 
## Problem

A simpler version of https://github.com/neondatabase/neon/pull/10812

## Summary of changes

Image layer creation will be preempted by L0 accumulated on other
timelines. We stop image layer generation if there's a pending L0
compaction request.

---------

Signed-off-by: Alex Chi Z <chi@neon.tech>
This commit is contained in:
Alex Chi Z.
2025-02-19 10:10:12 -05:00
committed by GitHub
parent aab5482fd5
commit 1f9511dbd9
1 changed files with 19 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
pageserver/src/tenant/timeline.rs
View File

@@ -22,6 +22,7 @@ use chrono::{DateTime, Utc};
use compaction::CompactionOutcome;
use enumset::EnumSet;
use fail::fail_point;
use futures::FutureExt;
use futures::{stream::FuturesUnordered, StreamExt};
use handle::ShardTimelineId;
use layer_manager::Shutdown;
@@ -5128,20 +5129,26 @@ impl Timeline {
// image layer generation taking too long time and blocking L0 compaction. So in this
// mode, we also inspect the current number of L0 layers and skip image layer generation
// if there are too many of them.
let num_of_l0_layers = {
let layers = self.layers.read().await;
layers.layer_map()?.level0_deltas().len()
};
let image_preempt_threshold = self.get_image_creation_preempt_threshold()
* self.get_compaction_threshold();
if image_preempt_threshold != 0 && num_of_l0_layers >= image_preempt_threshold {
tracing::info!(
"preempt image layer generation at {lsn} when processing partition {}..{}: too many L0 layers {}",
partition.start().unwrap(), partition.end().unwrap(), num_of_l0_layers
);
last_partition_processed = Some(partition.clone());
all_generated = false;
break;
// TODO: currently we do not respect `get_image_creation_preempt_threshold` and always yield
// when there is a single timeline with more than L0 threshold L0 layers. As long as the
// `get_image_creation_preempt_threshold` is set to a value greater than 0, we will yield for L0 compaction.
if image_preempt_threshold != 0 {
let should_yield = self
.l0_compaction_trigger
.notified()
.now_or_never()
.is_some();
if should_yield {
tracing::info!(
"preempt image layer generation at {lsn} when processing partition {}..{}: too many L0 layers",
partition.start().unwrap(), partition.end().unwrap()
);
last_partition_processed = Some(partition.clone());
all_generated = false;
break;
}
}
}
}