From c4b57e4b8fb55360bdb77cc9165be8fc31b0b469 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 7 Apr 2022 20:50:12 +0300 Subject: [PATCH] Move BlobRef It's not needed in image layers anymore, so move it into delta_layer.rs --- pageserver/src/layered_repository/blob_io.rs | 17 ++++++++++ pageserver/src/layered_repository/block_io.rs | 2 -- .../src/layered_repository/delta_layer.rs | 32 ++++++++++++++++++- .../src/layered_repository/image_layer.rs | 21 ++++++------ .../src/layered_repository/storage_layer.rs | 31 ------------------ 5 files changed, 57 insertions(+), 46 deletions(-) diff --git a/pageserver/src/layered_repository/blob_io.rs b/pageserver/src/layered_repository/blob_io.rs index 10bfea934d..aa90bbd0cf 100644 --- a/pageserver/src/layered_repository/blob_io.rs +++ b/pageserver/src/layered_repository/blob_io.rs @@ -10,12 +10,15 @@ use std::io::Error; /// For reading pub trait BlobCursor { + /// Read a blob into a new buffer. fn read_blob(&mut self, offset: u64) -> Result, std::io::Error> { let mut buf = Vec::new(); self.read_blob_into_buf(offset, &mut buf)?; Ok(buf) } + /// Read blob into the given buffer. Any previous contents in the buffer + /// are overwritten. fn read_blob_into_buf( &mut self, offset: u64, @@ -75,10 +78,19 @@ where } } +/// +/// Abstract trait for a data sink that you can write blobs to. +/// pub trait BlobWriter { + /// Write a blob of data. Returns the offset that it was written to, + /// which can be used to retrieve the data later. fn write_blob(&mut self, srcbuf: &[u8]) -> Result; } +/// +/// An implementation of BlobWriter to write blobs to anything that +/// implements std::io::Write. +/// pub struct WriteBlobWriter where W: std::io::Write, @@ -102,6 +114,11 @@ where self.offset } + /// Access the underlying Write object. + /// + /// NOTE: WriteBlobWriter keeps track of the current write offset. If + /// you write something directly to the inner Write object, it makes the + /// internally tracked 'offset' to go out of sync. So don't do that. pub fn into_inner(self) -> W { self.inner } diff --git a/pageserver/src/layered_repository/block_io.rs b/pageserver/src/layered_repository/block_io.rs index 2b8e31e1ee..a8992a6cb5 100644 --- a/pageserver/src/layered_repository/block_io.rs +++ b/pageserver/src/layered_repository/block_io.rs @@ -1,8 +1,6 @@ //! //! Low-level Block-oriented I/O functions //! -//! -//! use crate::page_cache; use crate::page_cache::{ReadBufResult, PAGE_SZ}; diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index f8828b541f..43122fd99d 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -35,7 +35,7 @@ use crate::layered_repository::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter use crate::layered_repository::block_io::{BlockCursor, BlockReader, FileBlockReader}; use crate::layered_repository::filename::{DeltaFileName, PathOrConf}; use crate::layered_repository::storage_layer::{ - BlobRef, Layer, ValueReconstructResult, ValueReconstructState, + Layer, ValueReconstructResult, ValueReconstructState, }; use crate::page_cache::{PageReadGuard, PAGE_SZ}; use crate::repository::{Key, Value}; @@ -93,6 +93,36 @@ impl From<&DeltaLayer> for Summary { } } +// Flag indicating that this version initialize the page +const WILL_INIT: u64 = 1; + +/// +/// Struct representing reference to BLOB in layers. Reference contains BLOB +/// offset, and for WAL records it also contains `will_init` flag. The flag +/// helps to determine the range of records that needs to be applied, without +/// reading/deserializing records themselves. +/// +#[derive(Debug, Serialize, Deserialize, Copy, Clone)] +struct BlobRef(u64); + +impl BlobRef { + pub fn will_init(&self) -> bool { + (self.0 & WILL_INIT) != 0 + } + + pub fn pos(&self) -> u64 { + self.0 >> 1 + } + + pub fn new(pos: u64, will_init: bool) -> BlobRef { + let mut blob_ref = pos << 1; + if will_init { + blob_ref |= WILL_INIT; + } + BlobRef(blob_ref) + } +} + /// /// DeltaLayer is the in-memory data structure associated with an /// on-disk delta file. We keep a DeltaLayer in memory for each diff --git a/pageserver/src/layered_repository/image_layer.rs b/pageserver/src/layered_repository/image_layer.rs index a8e5de09f5..d0afce1549 100644 --- a/pageserver/src/layered_repository/image_layer.rs +++ b/pageserver/src/layered_repository/image_layer.rs @@ -28,7 +28,7 @@ use crate::layered_repository::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter use crate::layered_repository::block_io::{BlockReader, FileBlockReader}; use crate::layered_repository::filename::{ImageFileName, PathOrConf}; use crate::layered_repository::storage_layer::{ - BlobRef, Layer, ValueReconstructResult, ValueReconstructState, + Layer, ValueReconstructResult, ValueReconstructState, }; use crate::page_cache::PAGE_SZ; use crate::repository::{Key, Value}; @@ -105,7 +105,7 @@ pub struct ImageLayerInner { loaded: bool, /// offset of each value - index: HashMap, + index: HashMap, // values copied from summary index_start_blk: u32, @@ -147,18 +147,18 @@ impl Layer for ImageLayer { assert!(lsn_range.end >= self.lsn); let inner = self.load()?; - if let Some(blob_ref) = inner.index.get(&key) { + if let Some(&offset) = inner.index.get(&key) { let buf = inner .file .as_ref() .unwrap() .block_cursor() - .read_blob(blob_ref.pos()) + .read_blob(offset) .with_context(|| { format!( "failed to read blob from data file {} at offset {}", self.filename().display(), - blob_ref.pos() + offset ) })?; let value = Bytes::from(buf); @@ -228,11 +228,8 @@ impl Layer for ImageLayer { let inner = self.load()?; - let mut index_vec: Vec<(&Key, &BlobRef)> = inner.index.iter().collect(); - index_vec.sort_by_key(|x| x.1.pos()); - - for (key, blob_ref) in index_vec { - println!("key: {} offset {}", key, blob_ref.pos()); + for (key, offset) in inner.index.iter() { + println!("key: {} offset {}", key, offset); } Ok(()) @@ -423,7 +420,7 @@ pub struct ImageLayerWriter { key_range: Range, lsn: Lsn, - index: HashMap, + index: HashMap, blob_writer: WriteBlobWriter, } @@ -476,7 +473,7 @@ impl ImageLayerWriter { ensure!(self.key_range.contains(&key)); let off = self.blob_writer.write_blob(img)?; - let old = self.index.insert(key, BlobRef::new(off, true)); + let old = self.index.insert(key, off); assert!(old.is_none()); Ok(()) diff --git a/pageserver/src/layered_repository/storage_layer.rs b/pageserver/src/layered_repository/storage_layer.rs index b5366da223..5ad43182f6 100644 --- a/pageserver/src/layered_repository/storage_layer.rs +++ b/pageserver/src/layered_repository/storage_layer.rs @@ -7,7 +7,6 @@ use crate::walrecord::ZenithWalRecord; use crate::{ZTenantId, ZTimelineId}; use anyhow::Result; use bytes::Bytes; -use serde::{Deserialize, Serialize}; use std::ops::Range; use std::path::PathBuf; @@ -145,33 +144,3 @@ pub trait Layer: Send + Sync { /// Dump summary of the contents of the layer to stdout fn dump(&self, verbose: bool) -> Result<()>; } - -// Flag indicating that this version initialize the page -const WILL_INIT: u64 = 1; - -/// -/// Struct representing reference to BLOB in layers. Reference contains BLOB -/// offset, and for WAL records it also contains `will_init` flag. The flag -/// helps to determine the range of records that needs to be applied, without -/// reading/deserializing records themselves. -/// -#[derive(Debug, Serialize, Deserialize, Copy, Clone)] -pub struct BlobRef(u64); - -impl BlobRef { - pub fn will_init(&self) -> bool { - (self.0 & WILL_INIT) != 0 - } - - pub fn pos(&self) -> u64 { - self.0 >> 1 - } - - pub fn new(pos: u64, will_init: bool) -> BlobRef { - let mut blob_ref = pos << 1; - if will_init { - blob_ref |= WILL_INIT; - } - BlobRef(blob_ref) - } -}