pageserver - fsync layerfiles after writing

Instead of fsyncing after each file is written, do all the fsyncs at the
end. On macOS I observed `zenith init` time go from 36 s -> 3 s.

pageserver/src/layered_repository.rs

@@ -1212,8 +1212,7 @@ impl LayeredTimeline {
         // a lot of memory and/or aren't receiving much updates anymore.
         let mut disk_consistent_lsn = last_record_lsn;
 
-        let mut created_historics = false;
-        let mut layer_uploads = Vec::new();
+        let mut layer_paths = Vec::new();
         while let Some((oldest_layer, oldest_generation)) = layers.peek_oldest_open() {
             let oldest_pending_lsn = oldest_layer.get_oldest_pending_lsn();
 
@@ -1259,20 +1258,16 @@ impl LayeredTimeline {
             write_guard = self.write_lock.lock().unwrap();
             layers = self.layers.lock().unwrap();
 
-            if !new_historics.is_empty() {
-                created_historics = true;
-            }
-
             // Finally, replace the frozen in-memory layer with the new on-disk layers
             layers.remove_historic(oldest_layer);
 
             // Add the historics to the LayerMap
             for delta_layer in new_historics.delta_layers {
-                layer_uploads.push(delta_layer.path());
+                layer_paths.push(delta_layer.path());
                 layers.insert_historic(Arc::new(delta_layer));
             }
             for image_layer in new_historics.image_layers {
-                layer_uploads.push(image_layer.path());
+                layer_paths.push(image_layer.path());
                 layers.insert_historic(Arc::new(image_layer));
             }
         }
@@ -1287,7 +1282,12 @@ impl LayeredTimeline {
         drop(layers);
         drop(write_guard);
 
-        if created_historics {
+        if !layer_paths.is_empty() {
+            for layer_path in &layer_paths {
+                let file = File::open(layer_path)?;
+                file.sync_all()?;
+            }
+
             // We must fsync the timeline dir to ensure the directory entries for
             // new layer files are durable
             let timeline_dir =
@@ -1330,7 +1330,7 @@ impl LayeredTimeline {
                 false,
             )?;
             if self.upload_relishes {
-                schedule_timeline_upload(self.tenantid, self.timelineid, layer_uploads, metadata);
+                schedule_timeline_upload(self.tenantid, self.timelineid, layer_paths, metadata);
             }
 
             // Also update the in-memory copy

pageserver/src/layered_repository/delta_layer.rs

@@ -431,8 +431,7 @@ impl DeltaLayer {
         let book = chapter.close()?;
 
         // This flushes the underlying 'buf_writer'.
-        let writer = book.close()?;
-        writer.get_ref().sync_all()?;
+        book.close()?;
 
         trace!("saved {}", &path.display());
 

pageserver/src/layered_repository/image_layer.rs

@@ -315,8 +315,7 @@ impl ImageLayer {
         let book = chapter.close()?;
 
         // This flushes the underlying 'buf_writer'.
-        let writer = book.close()?;
-        writer.get_ref().sync_all()?;
+        book.close()?;
 
         trace!("saved {}", path.display());
 

pageserver/src/layered_repository/inmemory_layer.rs

@@ -281,12 +281,6 @@ pub struct LayersOnDisk {
     pub image_layers: Vec<ImageLayer>,
 }
 
-impl LayersOnDisk {
-    pub fn is_empty(&self) -> bool {
-        self.delta_layers.is_empty() && self.image_layers.is_empty()
-    }
-}
-
 impl InMemoryLayer {
     /// Return the oldest page version that's stored in this layer
     pub fn get_oldest_pending_lsn(&self) -> Lsn {