From 1bb8ca08066aff53d2d1ac3e87b5fd9ff099a90b Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 6 Mar 2023 19:02:32 +0200 Subject: [PATCH] Split keyspace in partitions without holes --- pageserver/src/keyspace.rs | 11 +++- pageserver/src/tenant/timeline.rs | 100 ++++++++++++++---------------- 2 files changed, 58 insertions(+), 53 deletions(-) diff --git a/pageserver/src/keyspace.rs b/pageserver/src/keyspace.rs index 64024a2d8d..bbaa127e4c 100644 --- a/pageserver/src/keyspace.rs +++ b/pageserver/src/keyspace.rs @@ -10,6 +10,7 @@ pub struct KeySpace { /// Contiguous ranges of keys that belong to the key space. In key order, /// and with no overlap. pub ranges: Vec>, + pub start: Key, } impl KeySpace { @@ -22,16 +23,20 @@ impl KeySpace { let target_nblocks = (target_size / BLCKSZ as u64) as usize; let mut parts = Vec::new(); - let mut current_part = Vec::new(); + let mut current_part: Vec> = Vec::new(); let mut current_part_size: usize = 0; + let mut part_start = Key::MIN; for range in &self.ranges { // If appending the next contiguous range in the keyspace to the current // partition would cause it to be too large, start a new partition. let this_size = key_range_size(range) as usize; if current_part_size + this_size > target_nblocks && !current_part.is_empty() { + let part_end = current_part.last().unwrap().end; parts.push(KeySpace { ranges: current_part, + start: part_start, }); + part_start = part_end; current_part = Vec::new(); current_part_size = 0; } @@ -44,8 +49,10 @@ impl KeySpace { let next = start.add(target_nblocks as u32); parts.push(KeySpace { ranges: vec![start..next], + start: part_start, }); start = next; + part_start = next; remain_size -= target_nblocks } current_part.push(start..range.end); @@ -56,6 +63,7 @@ impl KeySpace { if !current_part.is_empty() { parts.push(KeySpace { ranges: current_part, + start: part_start, }); } @@ -126,6 +134,7 @@ impl KeySpaceAccum { } KeySpace { ranges: self.ranges, + start: Key::MIN, } } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 4f8dcba3e9..4e20320efd 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -2914,7 +2914,18 @@ impl Timeline { let mut max_deltas = 0; let wanted_image_layers = self.wanted_image_layers.lock().unwrap(); - + if let Some(wanted) = &*wanted_image_layers { + let img_range = partition.start..partition.ranges.last().unwrap().end; + if let Some((_start, end)) = wanted.range(..img_range.end).next_back() { + if *end > img_range.start { + info!( + "Force generation of layer {}-{} wanted by GC)", + img_range.start, img_range.end + ); + return Ok(true); + } + } + } for part_range in &partition.ranges { let image_coverage = layers.image_coverage(part_range, lsn)?; for (img_range, last_img) in image_coverage { @@ -2947,26 +2958,6 @@ impl Timeline { ); return Ok(true); } - if num_deltas > 0 { - if let Some(wanted) = &*wanted_image_layers { - if let Some((_start, end)) = wanted.range(..img_range.end).next_back() { - if *end > img_range.start { - info!( - "Force generation of layer {}-{} wanted by GC ({} deltas)", - img_range.start, img_range.end, num_deltas - ); - return Ok(true); - } else { - info!( - "Image {}-{} is not wanted", - img_range.start, img_range.end - ); - } - } - } - } else { - info!("No deltas for range {}-{}", img_range.start, img_range.end); - } } } } @@ -3003,6 +2994,7 @@ impl Timeline { let img_range = start..partition.ranges.last().unwrap().end; start = img_range.end; if force || self.time_for_new_image_layer(partition, lsn)? { + let img_range = partition.start..partition.ranges.last().unwrap().end; let mut image_layer_writer = ImageLayerWriter::new( self.conf, self.timeline_id, @@ -3053,6 +3045,8 @@ impl Timeline { image_layers.push(image_layer); } } + // All wanted layers are taken in account + *self.wanted_image_layers.lock().unwrap() = None; // Sync the new layer to disk before adding it to the layer map, to make sure // we don't garbage collect something based on the new layer, before it has @@ -3834,43 +3828,45 @@ impl Timeline { "keeping {} because it is the latest layer", l.filename().file_name() ); - let mut to_remove: Vec = Vec::new(); - let mut insert_new_range = true; - let start = l.get_key_range().start; - let mut end = l.get_key_range().end; + if l.is_incremental() { + let mut to_remove: Vec = Vec::new(); + let mut insert_new_range = true; + let start = l.get_key_range().start; + let mut end = l.get_key_range().end; - if start != Key::MIN || end != Key::MAX { // ignore L0 layers - // check if this range overlaps with exited ranges - let mut iter = wanted_image_layers.range_mut(..end); - while let Some((prev_start, prev_end)) = iter.next_back() { - if *prev_end >= start { - // two ranges overlap - if *prev_start <= start { - // combine with prev range - insert_new_range = false; - if *prev_end < end { - // extend prev range - *prev_end = end; - info!("Extend wanted image {}..{}", *prev_start, end); + if start != Key::MIN || end != Key::MAX { + // check if this range overlaps with exited ranges + let mut iter = wanted_image_layers.range_mut(..end); + while let Some((prev_start, prev_end)) = iter.next_back() { + if *prev_end >= start { + // two ranges overlap + if *prev_start <= start { + // combine with prev range + insert_new_range = false; + if *prev_end < end { + // extend prev range + *prev_end = end; + info!("Extend wanted image {}..{}", *prev_start, end); + } + break; + } else { + to_remove.push(*prev_start); + if *prev_end > end { + end = *prev_end; + } } - break; } else { - to_remove.push(*prev_start); - if *prev_end > end { - end = *prev_end; - } + break; } - } else { - break; } - } - for key in to_remove { - wanted_image_layers.remove(&key); - } - if insert_new_range { - info!("Wanted image {}..{}", start, end); - wanted_image_layers.insert(start, end); + for key in to_remove { + wanted_image_layers.remove(&key); + } + if insert_new_range { + info!("Wanted image {}..{}", start, end); + wanted_image_layers.insert(start, end); + } } } result.layers_not_updated += 1;