rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-07 13:32:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(pageserver): better gc_compaction_split heuristics (#10727)

Browse Source 
## Problem

close https://github.com/neondatabase/neon/issues/10213

`range_search` only returns the top-most layers that may satisfy the
search, so it doesn't include all layers that might be accessed (the
user needs to recursively call this function). We need to retrieve the
full layer map and find overlaps in order to have a correct heuristics
of the job split.

## Summary of changes

Retrieve all layers and find overlaps instead of doing `range_search`.
The patch also reduces the time holding the layer map read guard.

Signed-off-by: Alex Chi Z <chi@neon.tech>
This commit is contained in:
Alex Chi Z.
2025-02-10 14:33:34 -05:00
committed by GitHub
parent 8c4e94107d
commit b0c7ee0175
1 changed files with 20 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
pageserver/src/tenant/timeline/compaction.rs
View File

@@ -2259,8 +2259,11 @@ impl Timeline {
split_key_ranges.push((start, end));
}
split_key_ranges.sort();
let guard = self.layers.read().await;
let layer_map = guard.layer_map()?;
let all_layers = {
let guard = self.layers.read().await;
let layer_map = guard.layer_map()?;
layer_map.iter_historic_layers().collect_vec()
};
let mut current_start = None;
let ranges_num = split_key_ranges.len();
for (idx, (start, end)) in split_key_ranges.into_iter().enumerate() {
@@ -2272,14 +2275,23 @@ impl Timeline {
// We have already processed this partition.
continue;
}
let res = layer_map.range_search(start..end, compact_below_lsn);
let total_size = res.found.keys().map(|x| x.layer.file_size()).sum::<u64>();
let overlapping_layers = {
let mut desc = Vec::new();
for layer in all_layers.iter() {
if overlaps_with(&layer.get_key_range(), &(start..end))
&& layer.get_lsn_range().start <= compact_below_lsn
{
desc.push(layer.clone());
}
}
desc
};
let total_size = overlapping_layers.iter().map(|x| x.file_size).sum::<u64>();
if total_size > sub_compaction_max_job_size_mb * 1024 * 1024 || ranges_num == idx + 1 {
// Try to extend the compaction range so that we include at least one full layer file.
let extended_end = res
.found
.keys()
.map(|layer| layer.layer.key_range.end)
let extended_end = overlapping_layers
.iter()
.map(|layer| layer.key_range.end)
.min();
// It is possible that the search range does not contain any layer files when we reach the end of the loop.
// In this case, we simply use the specified key range end.
@@ -2306,7 +2318,6 @@ impl Timeline {
current_start = Some(end);
}
}
drop(guard);
Ok(compact_jobs)
}