diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 65851a166d..fa617f1a7d 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -11,6 +11,7 @@ use std::time::SystemTime; use tracing::Instrument; use utils::id::TimelineId; use utils::lsn::Lsn; +use utils::sync::gate::GateError; use utils::sync::heavier_once_cell; use crate::config::PageServerConf; @@ -488,6 +489,14 @@ impl Drop for LayerInner { None => return, }; + // We will only do I/O during drop if our Timeline's layer_gate is open: this avoids + // the risk that we would race with Timeline::shutdown and end up doing I/O to a timeline + // path for which the Timeline object has been torn down already. + let _gate_guard = match timeline.layer_gate.enter() { + Ok(g) => g, + Err(GateError::GateClosed) => return, + }; + // If timeline is alive, we can construct a span with IDs for this function. let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %timeline.tenant_shard_id.tenant_id, shard_id=%timeline.tenant_shard_id.shard_slug(), timeline_id = %timeline.timeline_id); let path = self.build_local_path(&timeline.tenant_shard_id, &timeline.timeline_id); diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index e9f47bbf23..729b8cedfb 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -313,6 +313,10 @@ pub struct Timeline { /// Gate to prevent shutdown completing while I/O is still happening to this timeline's data pub(crate) gate: Gate, + /// Gate to prevent shutdown completing until all Layers for this Timeline have finished + /// doing any background I/O such as deleting files on drop. + pub(crate) layer_gate: Gate, + /// Cancellation token scoped to this timeline: anything doing long-running work relating /// to the timeline should drop out when this token fires. pub(crate) cancel: CancellationToken, @@ -1002,8 +1006,15 @@ impl Timeline { ) .await; - // Finally wait until any gate-holders are complete + // Wait until any normal gate-holders such as page_service requests are complete self.gate.close().await; + + // Drop our references to layers: this should permit all layers to be dropped, and any I/O + // in their drop() method to complete. + self.layers.write().await.clear(); + + // Wait until any Layer gate holders such as LayerInner::drop are complete + self.layer_gate.close().await; } pub fn set_state(&self, new_state: TimelineState) { @@ -1445,6 +1456,7 @@ impl Timeline { cancel, gate: Gate::new(format!("Timeline<{tenant_shard_id}/{timeline_id}>")), + layer_gate: Gate::new(format!("TimelineLayers<{tenant_shard_id}/{timeline_id}>")), compaction_lock: tokio::sync::Mutex::default(), gc_lock: tokio::sync::Mutex::default(), diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index dcd82949dd..10588ee884 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -33,6 +33,11 @@ impl LayerManager { } } + pub(crate) fn clear(&mut self) { + std::mem::take(&mut self.layer_map); + self.layer_fmgr.clear(); + } + pub(crate) fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Layer { self.layer_fmgr.get_from_desc(desc) } @@ -271,6 +276,10 @@ impl LayerFileManager { } } + pub(crate) fn clear(&mut self) { + self.0.clear(); + } + pub(crate) fn contains(&self, layer: &T) -> bool { self.0.contains_key(&layer.layer_desc().key()) }