Move BlobRef

It's not needed in image layers anymore, so move it into delta_layer.rs

pageserver/src/layered_repository/blob_io.rs

@@ -10,12 +10,15 @@ use std::io::Error;
 
 /// For reading
 pub trait BlobCursor {
+    /// Read a blob into a new buffer.
     fn read_blob(&mut self, offset: u64) -> Result<Vec<u8>, std::io::Error> {
         let mut buf = Vec::new();
         self.read_blob_into_buf(offset, &mut buf)?;
         Ok(buf)
     }
 
+    /// Read blob into the given buffer. Any previous contents in the buffer
+    /// are overwritten.
     fn read_blob_into_buf(
         &mut self,
         offset: u64,
@@ -75,10 +78,19 @@ where
     }
 }
 
+///
+/// Abstract trait for a data sink that you can write blobs to.
+///
 pub trait BlobWriter {
+    /// Write a blob of data. Returns the offset that it was written to,
+    /// which can be used to retrieve the data later.
     fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error>;
 }
 
+///
+/// An implementation of BlobWriter to write blobs to anything that
+/// implements std::io::Write.
+///
 pub struct WriteBlobWriter<W>
 where
     W: std::io::Write,
@@ -102,6 +114,11 @@ where
         self.offset
     }
 
+    /// Access the underlying Write object.
+    ///
+    /// NOTE: WriteBlobWriter keeps track of the current write offset. If
+    /// you write something directly to the inner Write object, it makes the
+    /// internally tracked 'offset' to go out of sync. So don't do that.
     pub fn into_inner(self) -> W {
         self.inner
     }

pageserver/src/layered_repository/block_io.rs

@@ -1,8 +1,6 @@
 //!
 //! Low-level Block-oriented I/O functions
 //!
-//!
-//!
 
 use crate::page_cache;
 use crate::page_cache::{ReadBufResult, PAGE_SZ};

pageserver/src/layered_repository/delta_layer.rs

@@ -35,7 +35,7 @@ use crate::layered_repository::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter
 use crate::layered_repository::block_io::{BlockCursor, BlockReader, FileBlockReader};
 use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    BlobRef, Layer, ValueReconstructResult, ValueReconstructState,
+    Layer, ValueReconstructResult, ValueReconstructState,
 };
 use crate::page_cache::{PageReadGuard, PAGE_SZ};
 use crate::repository::{Key, Value};
@@ -93,6 +93,36 @@ impl From<&DeltaLayer> for Summary {
     }
 }
 
+// Flag indicating that this version initialize the page
+const WILL_INIT: u64 = 1;
+
+///
+/// Struct representing reference to BLOB in layers. Reference contains BLOB
+/// offset, and for WAL records it also contains `will_init` flag. The flag
+/// helps to determine the range of records that needs to be applied, without
+/// reading/deserializing records themselves.
+///
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+struct BlobRef(u64);
+
+impl BlobRef {
+    pub fn will_init(&self) -> bool {
+        (self.0 & WILL_INIT) != 0
+    }
+
+    pub fn pos(&self) -> u64 {
+        self.0 >> 1
+    }
+
+    pub fn new(pos: u64, will_init: bool) -> BlobRef {
+        let mut blob_ref = pos << 1;
+        if will_init {
+            blob_ref |= WILL_INIT;
+        }
+        BlobRef(blob_ref)
+    }
+}
+
 ///
 /// DeltaLayer is the in-memory data structure associated with an
 /// on-disk delta file.  We keep a DeltaLayer in memory for each

pageserver/src/layered_repository/image_layer.rs

@@ -28,7 +28,7 @@ use crate::layered_repository::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter
 use crate::layered_repository::block_io::{BlockReader, FileBlockReader};
 use crate::layered_repository::filename::{ImageFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    BlobRef, Layer, ValueReconstructResult, ValueReconstructState,
+    Layer, ValueReconstructResult, ValueReconstructState,
 };
 use crate::page_cache::PAGE_SZ;
 use crate::repository::{Key, Value};
@@ -105,7 +105,7 @@ pub struct ImageLayerInner {
     loaded: bool,
 
     /// offset of each value
-    index: HashMap<Key, BlobRef>,
+    index: HashMap<Key, u64>,
 
     // values copied from summary
     index_start_blk: u32,
@@ -147,18 +147,18 @@ impl Layer for ImageLayer {
         assert!(lsn_range.end >= self.lsn);
 
         let inner = self.load()?;
-        if let Some(blob_ref) = inner.index.get(&key) {
+        if let Some(&offset) = inner.index.get(&key) {
             let buf = inner
                 .file
                 .as_ref()
                 .unwrap()
                 .block_cursor()
-                .read_blob(blob_ref.pos())
+                .read_blob(offset)
                 .with_context(|| {
                     format!(
                         "failed to read blob from data file {} at offset {}",
                         self.filename().display(),
-                        blob_ref.pos()
+                        offset
                     )
                 })?;
             let value = Bytes::from(buf);
@@ -228,11 +228,8 @@ impl Layer for ImageLayer {
 
         let inner = self.load()?;
 
-        let mut index_vec: Vec<(&Key, &BlobRef)> = inner.index.iter().collect();
-        index_vec.sort_by_key(|x| x.1.pos());
-
-        for (key, blob_ref) in index_vec {
-            println!("key: {} offset {}", key, blob_ref.pos());
+        for (key, offset) in inner.index.iter() {
+            println!("key: {} offset {}", key, offset);
         }
 
         Ok(())
@@ -423,7 +420,7 @@ pub struct ImageLayerWriter {
     key_range: Range<Key>,
     lsn: Lsn,
 
-    index: HashMap<Key, BlobRef>,
+    index: HashMap<Key, u64>,
 
     blob_writer: WriteBlobWriter<VirtualFile>,
 }
@@ -476,7 +473,7 @@ impl ImageLayerWriter {
         ensure!(self.key_range.contains(&key));
         let off = self.blob_writer.write_blob(img)?;
 
-        let old = self.index.insert(key, BlobRef::new(off, true));
+        let old = self.index.insert(key, off);
         assert!(old.is_none());
 
         Ok(())

pageserver/src/layered_repository/storage_layer.rs

@@ -7,7 +7,6 @@ use crate::walrecord::ZenithWalRecord;
 use crate::{ZTenantId, ZTimelineId};
 use anyhow::Result;
 use bytes::Bytes;
-use serde::{Deserialize, Serialize};
 use std::ops::Range;
 use std::path::PathBuf;
 
@@ -145,33 +144,3 @@ pub trait Layer: Send + Sync {
     /// Dump summary of the contents of the layer to stdout
     fn dump(&self, verbose: bool) -> Result<()>;
 }
-
-// Flag indicating that this version initialize the page
-const WILL_INIT: u64 = 1;
-
-///
-/// Struct representing reference to BLOB in layers. Reference contains BLOB
-/// offset, and for WAL records it also contains `will_init` flag. The flag
-/// helps to determine the range of records that needs to be applied, without
-/// reading/deserializing records themselves.
-///
-#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
-pub struct BlobRef(u64);
-
-impl BlobRef {
-    pub fn will_init(&self) -> bool {
-        (self.0 & WILL_INIT) != 0
-    }
-
-    pub fn pos(&self) -> u64 {
-        self.0 >> 1
-    }
-
-    pub fn new(pos: u64, will_init: bool) -> BlobRef {
-        let mut blob_ref = pos << 1;
-        if will_init {
-            blob_ref |= WILL_INIT;
-        }
-        BlobRef(blob_ref)
-    }
-}