pageserver: make Timeline::shutdown safe against Layer::drop

pageserver/src/tenant/storage_layer/layer.rs

@@ -11,6 +11,7 @@ use std::time::SystemTime;
 use tracing::Instrument;
 use utils::id::TimelineId;
 use utils::lsn::Lsn;
+use utils::sync::gate::GateError;
 use utils::sync::heavier_once_cell;
 
 use crate::config::PageServerConf;
@@ -488,6 +489,14 @@ impl Drop for LayerInner {
             None => return,
         };
 
+        // We will only do I/O during drop if our Timeline's layer_gate is open: this avoids
+        // the risk that we would race with Timeline::shutdown and end up doing I/O to a timeline
+        // path for which the Timeline object has been torn down already.
+        let _gate_guard = match timeline.layer_gate.enter() {
+            Ok(g) => g,
+            Err(GateError::GateClosed) => return,
+        };
+
         // If timeline is alive, we can construct a span with IDs for this function.
         let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %timeline.tenant_shard_id.tenant_id, shard_id=%timeline.tenant_shard_id.shard_slug(), timeline_id = %timeline.timeline_id);
         let path = self.build_local_path(&timeline.tenant_shard_id, &timeline.timeline_id);

pageserver/src/tenant/timeline.rs

@@ -313,6 +313,10 @@ pub struct Timeline {
     /// Gate to prevent shutdown completing while I/O is still happening to this timeline's data
     pub(crate) gate: Gate,
 
+    /// Gate to prevent shutdown completing until all Layers for this Timeline have finished
+    /// doing any background I/O such as deleting files on drop.
+    pub(crate) layer_gate: Gate,
+
     /// Cancellation token scoped to this timeline: anything doing long-running work relating
     /// to the timeline should drop out when this token fires.
     pub(crate) cancel: CancellationToken,
@@ -1002,8 +1006,15 @@ impl Timeline {
         )
         .await;
 
-        // Finally wait until any gate-holders are complete
+        // Wait until any normal gate-holders such as page_service requests are complete
         self.gate.close().await;
+
+        // Drop our references to layers: this should permit all layers to be dropped, and any I/O
+        // in their drop() method to complete.
+        self.layers.write().await.clear();
+
+        // Wait until any Layer gate holders such as LayerInner::drop are complete
+        self.layer_gate.close().await;
     }
 
     pub fn set_state(&self, new_state: TimelineState) {
@@ -1445,6 +1456,7 @@ impl Timeline {
 
                 cancel,
                 gate: Gate::new(format!("Timeline<{tenant_shard_id}/{timeline_id}>")),
+                layer_gate: Gate::new(format!("TimelineLayers<{tenant_shard_id}/{timeline_id}>")),
 
                 compaction_lock: tokio::sync::Mutex::default(),
                 gc_lock: tokio::sync::Mutex::default(),

pageserver/src/tenant/timeline/layer_manager.rs

@@ -33,6 +33,11 @@ impl LayerManager {
         }
     }
 
+    pub(crate) fn clear(&mut self) {
+        std::mem::take(&mut self.layer_map);
+        self.layer_fmgr.clear();
+    }
+
     pub(crate) fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Layer {
         self.layer_fmgr.get_from_desc(desc)
     }
@@ -271,6 +276,10 @@ impl<T: AsLayerDesc + Clone> LayerFileManager<T> {
         }
     }
 
+    pub(crate) fn clear(&mut self) {
+        self.0.clear();
+    }
+
     pub(crate) fn contains(&self, layer: &T) -> bool {
         self.0.contains_key(&layer.layer_desc().key())
     }