From ea4c3639e342625ffa4523a37d7229210b27aa33 Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Thu, 16 Sep 2021 21:32:05 -0700 Subject: [PATCH] Include layer metadata in layer summary chapters Include all data stored in layer filenames and the tenant+timeline IDs inside a summary chapter. Use this chapter in the `dump_layerfile` utility. --- pageserver/src/layered_repository.rs | 23 ++-- .../src/layered_repository/delta_layer.rs | 101 +++++++++++++++--- .../src/layered_repository/image_layer.rs | 92 +++++++++++++--- .../src/layered_repository/storage_layer.rs | 2 +- 4 files changed, 172 insertions(+), 46 deletions(-) diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 3634ce2e7f..eb0e4368f2 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -12,6 +12,7 @@ //! use anyhow::{anyhow, bail, Context, Result}; +use bookfile::Book; use bytes::Bytes; use lazy_static::lazy_static; use log::*; @@ -25,7 +26,6 @@ use std::fs::File; use std::io::Write; use std::ops::Bound::Included; use std::path::{Path, PathBuf}; -use std::str::FromStr; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; @@ -57,7 +57,6 @@ mod storage_layer; use delta_layer::DeltaLayer; use image_layer::ImageLayer; -use filename::{DeltaFileName, ImageFileName}; use inmemory_layer::InMemoryLayer; use layer_map::LayerMap; use storage_layer::{ @@ -1716,17 +1715,17 @@ impl LayeredTimeline { /// Dump contents of a layer file to stdout. pub fn dump_layerfile_from_path(path: &Path) -> Result<()> { - let fname = path.file_name().unwrap().to_str().unwrap(); + let file = File::open(path)?; + let book = Book::new(file)?; - let dummy_tenantid = ZTenantId::from_str("00000000000000000000000000000000")?; - let dummy_timelineid = ZTimelineId::from_str("00000000000000000000000000000000")?; - - if let Some(deltafilename) = DeltaFileName::from_str(fname) { - DeltaLayer::new_for_path(path, dummy_timelineid, dummy_tenantid, &deltafilename).dump()?; - } else if let Some(imgfilename) = ImageFileName::from_str(fname) { - ImageLayer::new_for_path(path, dummy_timelineid, dummy_tenantid, &imgfilename).dump()?; - } else { - bail!("unrecognized layer file name : {}", fname); + match book.magic() { + delta_layer::DELTA_FILE_MAGIC => { + DeltaLayer::new_for_path(path, &book)?.dump()?; + } + image_layer::IMAGE_FILE_MAGIC => { + ImageLayer::new_for_path(path, &book)?.dump()?; + } + magic => bail!("unrecognized magic identifier: {:?}", magic), } Ok(()) diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index 03f9b0350f..cda2f33e6f 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -69,7 +69,7 @@ use zenith_utils::lsn::Lsn; use super::blob::{read_blob, BlobRange}; // Magic constant to identify a Zenith delta file -static DELTA_FILE_MAGIC: u32 = 0x5A616E01; +pub const DELTA_FILE_MAGIC: u32 = 0x5A616E01; /// Mapping from (block #, lsn) -> page/WAL record /// byte ranges in PAGE_VERSIONS_CHAPTER @@ -79,6 +79,36 @@ static PAGE_VERSION_METAS_CHAPTER: u64 = 1; static PAGE_VERSIONS_CHAPTER: u64 = 2; static REL_SIZES_CHAPTER: u64 = 3; +/// Contains the [`Summary`] struct +static SUMMARY_CHAPTER: u64 = 4; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct Summary { + tenantid: ZTenantId, + timelineid: ZTimelineId, + seg: SegmentTag, + + start_lsn: Lsn, + end_lsn: Lsn, + + dropped: bool, +} + +impl From<&DeltaLayer> for Summary { + fn from(layer: &DeltaLayer) -> Self { + Self { + tenantid: layer.tenantid, + timelineid: layer.timelineid, + seg: layer.seg, + + start_lsn: layer.start_lsn, + end_lsn: layer.end_lsn, + + dropped: layer.dropped, + } + } +} + #[derive(Serialize, Deserialize)] struct PageVersionMeta { page_image_range: Option, @@ -298,8 +328,8 @@ impl Layer for DeltaLayer { /// debugging function to print out the contents of the layer fn dump(&self) -> Result<()> { println!( - "----- delta layer for tli {} seg {} {}-{} ----", - self.timelineid, self.seg, self.start_lsn, self.end_lsn + "----- delta layer for ten {} tli {} seg {} {}-{} ----", + self.tenantid, self.timelineid, self.seg, self.start_lsn, self.end_lsn ); println!("--- relsizes ---"); @@ -438,6 +468,20 @@ impl DeltaLayer { chapter.write_all(&buf)?; let book = chapter.close()?; + let mut chapter = book.new_chapter(SUMMARY_CHAPTER); + let summary = Summary { + tenantid, + timelineid, + seg, + + start_lsn, + end_lsn, + + dropped, + }; + Summary::ser_into(&summary, &mut chapter)?; + let book = chapter.close()?; + book.close()?; trace!("saved {}", &path.display()); @@ -479,6 +523,31 @@ impl DeltaLayer { let (path, book) = self.open_book()?; + match &self.path_or_conf { + PathOrConf::Conf(_) => { + let chapter = book.read_chapter(SUMMARY_CHAPTER)?; + let actual_summary = Summary::des(&chapter)?; + + let expected_summary = Summary::from(self); + + if actual_summary != expected_summary { + bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary); + } + } + PathOrConf::Path(path) => { + let actual_filename = Path::new(path.file_name().unwrap()); + let expected_filename = self.filename(); + + if actual_filename != expected_filename { + println!( + "warning: filename does not match what is expected from in-file summary" + ); + println!("actual: {:?}", actual_filename); + println!("expected: {:?}", expected_filename); + } + } + } + let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?; let page_version_metas = BTreeMap::des(&chapter)?; @@ -524,26 +593,24 @@ impl DeltaLayer { /// Create a DeltaLayer struct representing an existing file on disk. /// /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary. - pub fn new_for_path( - path: &Path, - timelineid: ZTimelineId, - tenantid: ZTenantId, - filename: &DeltaFileName, - ) -> DeltaLayer { - DeltaLayer { + pub fn new_for_path(path: &Path, book: &Book) -> Result { + let chapter = book.read_chapter(SUMMARY_CHAPTER)?; + let summary = Summary::des(&chapter)?; + + Ok(DeltaLayer { path_or_conf: PathOrConf::Path(path.to_path_buf()), - timelineid, - tenantid, - seg: filename.seg, - start_lsn: filename.start_lsn, - end_lsn: filename.end_lsn, - dropped: filename.dropped, + timelineid: summary.timelineid, + tenantid: summary.tenantid, + seg: summary.seg, + start_lsn: summary.start_lsn, + end_lsn: summary.end_lsn, + dropped: summary.dropped, inner: Mutex::new(DeltaLayerInner { loaded: false, page_version_metas: BTreeMap::new(), relsizes: BTreeMap::new(), }), predecessor: None, - } + }) } } diff --git a/pageserver/src/layered_repository/image_layer.rs b/pageserver/src/layered_repository/image_layer.rs index 4fdd7dc45c..a3abd7bed9 100644 --- a/pageserver/src/layered_repository/image_layer.rs +++ b/pageserver/src/layered_repository/image_layer.rs @@ -29,9 +29,10 @@ use crate::layered_repository::LayeredTimeline; use crate::layered_repository::RELISH_SEG_SIZE; use crate::PageServerConf; use crate::{ZTenantId, ZTimelineId}; -use anyhow::{anyhow, ensure, Result}; +use anyhow::{anyhow, bail, ensure, Result}; use bytes::Bytes; use log::*; +use serde::{Deserialize, Serialize}; use std::convert::TryInto; use std::fs; use std::fs::File; @@ -41,15 +42,40 @@ use std::sync::{Mutex, MutexGuard}; use bookfile::{Book, BookWriter}; +use zenith_utils::bin_ser::BeSer; use zenith_utils::lsn::Lsn; // Magic constant to identify a Zenith segment image file -const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1; +pub const IMAGE_FILE_MAGIC: u32 = 0x5A616E01 + 1; /// Contains each block in block # order const BLOCKY_IMAGES_CHAPTER: u64 = 1; const NONBLOCKY_IMAGE_CHAPTER: u64 = 2; +/// Contains the [`Summary`] struct +const SUMMARY_CHAPTER: u64 = 3; + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +struct Summary { + tenantid: ZTenantId, + timelineid: ZTimelineId, + seg: SegmentTag, + + lsn: Lsn, +} + +impl From<&ImageLayer> for Summary { + fn from(layer: &ImageLayer) -> Self { + Self { + tenantid: layer.tenantid, + timelineid: layer.timelineid, + seg: layer.seg, + + lsn: layer.lsn, + } + } +} + const BLOCK_SIZE: usize = 8192; /// @@ -208,8 +234,8 @@ impl Layer for ImageLayer { /// debugging function to print out the contents of the layer fn dump(&self) -> Result<()> { println!( - "----- image layer for tli {} seg {} at {} ----", - self.timelineid, self.seg, self.lsn + "----- image layer for ten {} tli {} seg {} at {} ----", + self.tenantid, self.timelineid, self.seg, self.lsn ); let inner = self.load()?; @@ -297,6 +323,17 @@ impl ImageLayer { } }; + let mut chapter = book.new_chapter(SUMMARY_CHAPTER); + let summary = Summary { + tenantid, + timelineid, + seg, + + lsn, + }; + Summary::ser_into(&summary, &mut chapter)?; + let book = chapter.close()?; + book.close()?; trace!("saved {}", &path.display()); @@ -357,6 +394,31 @@ impl ImageLayer { let (path, book) = self.open_book()?; + match &self.path_or_conf { + PathOrConf::Conf(_) => { + let chapter = book.read_chapter(SUMMARY_CHAPTER)?; + let actual_summary = Summary::des(&chapter)?; + + let expected_summary = Summary::from(self); + + if actual_summary != expected_summary { + bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary); + } + } + PathOrConf::Path(path) => { + let actual_filename = Path::new(path.file_name().unwrap()); + let expected_filename = self.filename(); + + if actual_filename != expected_filename { + println!( + "warning: filename does not match what is expected from in-file summary" + ); + println!("actual: {:?}", actual_filename); + println!("expected: {:?}", expected_filename); + } + } + } + let image_type = if self.seg.rel.is_blocky() { let chapter = book.chapter_reader(BLOCKY_IMAGES_CHAPTER)?; let images_len = chapter.len(); @@ -418,22 +480,20 @@ impl ImageLayer { /// Create an ImageLayer struct representing an existing file on disk. /// /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary. - pub fn new_for_path( - path: &Path, - timelineid: ZTimelineId, - tenantid: ZTenantId, - filename: &ImageFileName, - ) -> ImageLayer { - ImageLayer { + pub fn new_for_path(path: &Path, book: &Book) -> Result { + let chapter = book.read_chapter(SUMMARY_CHAPTER)?; + let summary = Summary::des(&chapter)?; + + Ok(ImageLayer { path_or_conf: PathOrConf::Path(path.to_path_buf()), - timelineid, - tenantid, - seg: filename.seg, - lsn: filename.lsn, + timelineid: summary.timelineid, + tenantid: summary.tenantid, + seg: summary.seg, + lsn: summary.lsn, inner: Mutex::new(ImageLayerInner { loaded: false, image_type: ImageType::Blocky { num_blocks: 0 }, }), - } + }) } } diff --git a/pageserver/src/layered_repository/storage_layer.rs b/pageserver/src/layered_repository/storage_layer.rs index 48bc87e1c5..73207514e0 100644 --- a/pageserver/src/layered_repository/storage_layer.rs +++ b/pageserver/src/layered_repository/storage_layer.rs @@ -21,7 +21,7 @@ pub const RELISH_SEG_SIZE: u32 = 10 * 1024 * 1024 / 8192; /// Each relish stored in the repository is divided into fixed-sized "segments", /// with 10 MB of key-space, or 1280 8k pages each. /// -#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Clone, Copy, Serialize, Deserialize)] pub struct SegmentTag { pub rel: RelishTag, pub segno: u32,