From a91eeb1c6555a5ff5cab48236fd22c429870fda6 Mon Sep 17 00:00:00 2001
From: Heikki Linnakangas <heikki@zenith.tech>
Date: Tue, 21 Sep 2021 16:54:29 +0300
Subject: [PATCH] Buffer the writes when writing a layer to disk.

Significantly reduces the CPU time spent on libc::write.
---
 pageserver/src/layered_repository/blob.rs        | 10 +++++-----
 pageserver/src/layered_repository/delta_layer.rs |  6 ++++--
 pageserver/src/layered_repository/image_layer.rs |  6 ++++--
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/pageserver/src/layered_repository/blob.rs b/pageserver/src/layered_repository/blob.rs
index 252920e33e..b7c7c3f460 100644
--- a/pageserver/src/layered_repository/blob.rs
+++ b/pageserver/src/layered_repository/blob.rs
@@ -16,14 +16,14 @@ pub fn read_blob(reader: &BoundedReader<&'_ File>, range: &BlobRange) -> Result<
     Ok(buf)
 }
 
-pub struct BlobWriter {
-    writer: ChapterWriter<File>,
+pub struct BlobWriter<W> {
+    writer: ChapterWriter<W>,
     offset: u64,
 }
 
-impl BlobWriter {
+impl<W: Write> BlobWriter<W> {
     // This function takes a BookWriter and creates a new chapter to ensure offset is 0.
-    pub fn new(book_writer: BookWriter<File>, chapter_id: impl Into<ChapterId>) -> Self {
+    pub fn new(book_writer: BookWriter<W>, chapter_id: impl Into<ChapterId>) -> Self {
         let writer = book_writer.new_chapter(chapter_id);
         Self { writer, offset: 0 }
     }
@@ -39,7 +39,7 @@ impl BlobWriter {
         Ok(range)
     }
 
-    pub fn close(self) -> bookfile::Result<BookWriter<File>> {
+    pub fn close(self) -> bookfile::Result<BookWriter<W>> {
         self.writer.close()
     }
 }
diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs
index cda2f33e6f..b834daae10 100644
--- a/pageserver/src/layered_repository/delta_layer.rs
+++ b/pageserver/src/layered_repository/delta_layer.rs
@@ -56,7 +56,7 @@ use std::collections::BTreeMap;
 use std::fmt::Write as _;
 use std::fs;
 use std::fs::File;
-use std::io::Write;
+use std::io::{BufWriter, Write};
 use std::ops::Bound::Included;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex, MutexGuard};
@@ -425,7 +425,8 @@ impl DeltaLayer {
         // Note: This overwrites any existing file. There shouldn't be any.
         // FIXME: throw an error instead?
         let file = File::create(&path)?;
-        let book = BookWriter::new(file, DELTA_FILE_MAGIC)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, DELTA_FILE_MAGIC)?;
 
         let mut page_version_writer = BlobWriter::new(book, PAGE_VERSIONS_CHAPTER);
 
@@ -482,6 +483,7 @@ impl DeltaLayer {
         Summary::ser_into(&summary, &mut chapter)?;
         let book = chapter.close()?;
 
+        // This flushes the underlying 'buf_writer'.
         book.close()?;
 
         trace!("saved {}", &path.display());
diff --git a/pageserver/src/layered_repository/image_layer.rs b/pageserver/src/layered_repository/image_layer.rs
index a3abd7bed9..2ad1853755 100644
--- a/pageserver/src/layered_repository/image_layer.rs
+++ b/pageserver/src/layered_repository/image_layer.rs
@@ -36,7 +36,7 @@ use serde::{Deserialize, Serialize};
 use std::convert::TryInto;
 use std::fs;
 use std::fs::File;
-use std::io::Write;
+use std::io::{BufWriter, Write};
 use std::path::{Path, PathBuf};
 use std::sync::{Mutex, MutexGuard};
 
@@ -305,7 +305,8 @@ impl ImageLayer {
         // Note: This overwrites any existing file. There shouldn't be any.
         // FIXME: throw an error instead?
         let file = File::create(&path)?;
-        let book = BookWriter::new(file, IMAGE_FILE_MAGIC)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;
 
         let book = match &image_type {
             ImageType::Blocky { .. } => {
@@ -334,6 +335,7 @@ impl ImageLayer {
         Summary::ser_into(&summary, &mut chapter)?;
         let book = chapter.close()?;
 
+        // This flushes the underlying 'buf_writer'.
         book.close()?;
 
         trace!("saved {}", &path.display());