From 8c77ccfc01bf5e634bf2adab9543a2f2e1b9420e Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 15 Apr 2025 09:25:09 +0200 Subject: [PATCH] pageserver: log total progress during shard ancestor compaction (#11565) ## Problem Shard ancestor compaction doesn't currently log any global progress information, only for the current batch. ## Summary of changes Log the number of layers checked for eligibility this iteration, and the total number of layers to check. This will indicate how far along the total shard ancestor compaction has gotten for this iteration. --- pageserver/src/tenant/layer_map.rs | 2 +- .../src/tenant/layer_map/historic_layer_coverage.rs | 2 +- pageserver/src/tenant/timeline/compaction.rs | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index 96cee922ff..23052ccee7 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -714,7 +714,7 @@ impl LayerMap { true } - pub fn iter_historic_layers(&self) -> impl '_ + Iterator> { + pub fn iter_historic_layers(&self) -> impl ExactSizeIterator> { self.historic.iter() } diff --git a/pageserver/src/tenant/layer_map/historic_layer_coverage.rs b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs index b3dc8e56a3..5ccc75fff6 100644 --- a/pageserver/src/tenant/layer_map/historic_layer_coverage.rs +++ b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs @@ -504,7 +504,7 @@ impl BufferedHistoricLayerCoverage { } /// Iterate all the layers - pub fn iter(&self) -> impl '_ + Iterator { + pub fn iter(&self) -> impl ExactSizeIterator { // NOTE we can actually perform this without rebuilding, // but it's not necessary for now. if !self.buffer.is_empty() { diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 91cc8ca10c..2a450795fd 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -1273,7 +1273,10 @@ impl Timeline { let pitr_cutoff = self.gc_info.read().unwrap().cutoffs.time; let layers = self.layers.read().await; - for layer_desc in layers.layer_map()?.iter_historic_layers() { + let layers_iter = layers.layer_map()?.iter_historic_layers(); + let (layers_total, mut layers_checked) = (layers_iter.len(), 0); + for layer_desc in layers_iter { + layers_checked += 1; let layer = layers.get_from_desc(&layer_desc); if layer.metadata().shard.shard_count == self.shard_identity.count { // This layer does not belong to a historic ancestor, no need to re-image it. @@ -1371,7 +1374,8 @@ impl Timeline { } info!( - "starting shard ancestor compaction, rewriting {} layers and dropping {} layers \ + "starting shard ancestor compaction, rewriting {} layers and dropping {} layers, \ + checked {layers_checked}/{layers_total} layers \ (latest_gc_cutoff={} pitr_cutoff={})", layers_to_rewrite.len(), drop_layers.len(),