From 40c79988a80be1aca3742e24056b4756d1b67d88 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Wed, 25 Aug 2021 17:57:50 +0300 Subject: [PATCH] Move code to handle snapshot filenames This isn't very useful yet, but the next commit will add more code related to handling the filenames. --- pageserver/src/layered_repository.rs | 19 +- pageserver/src/layered_repository/filename.rs | 172 ++++++++++++++++ .../src/layered_repository/snapshot_layer.rs | 190 ++---------------- 3 files changed, 203 insertions(+), 178 deletions(-) create mode 100644 pageserver/src/layered_repository/filename.rs diff --git a/pageserver/src/layered_repository.rs b/pageserver/src/layered_repository.rs index 1dc5cec82f..1ad2aff1ce 100644 --- a/pageserver/src/layered_repository.rs +++ b/pageserver/src/layered_repository.rs @@ -39,6 +39,7 @@ use zenith_utils::bin_ser::BeSer; use zenith_utils::lsn::{AtomicLsn, Lsn}; use zenith_utils::seqwait::SeqWait; +mod filename; mod inmemory_layer; mod layer_map; mod snapshot_layer; @@ -893,19 +894,21 @@ impl LayeredTimeline { self.timelineid ); let mut layers = self.layers.lock().unwrap(); - let snapfiles = - SnapshotLayer::list_snapshot_files(self.conf, self.timelineid, self.tenantid)?; + let snapfilenames = + filename::list_snapshot_files(self.conf, self.timelineid, self.tenantid)?; + + for filename in snapfilenames.iter() { + let layer = SnapshotLayer::load_snapshot_layer(self.conf, self.timelineid, self.tenantid, filename)?; - for layer_rc in snapfiles.iter() { info!( "found layer {} {}-{} {} on timeline {}", - layer_rc.get_seg_tag(), - layer_rc.get_start_lsn(), - layer_rc.get_end_lsn(), - layer_rc.is_dropped(), + layer.get_seg_tag(), + layer.get_start_lsn(), + layer.get_end_lsn(), + layer.is_dropped(), self.timelineid ); - layers.insert_historic(Arc::clone(layer_rc)); + layers.insert_historic(Arc::new(layer)); } Ok(()) diff --git a/pageserver/src/layered_repository/filename.rs b/pageserver/src/layered_repository/filename.rs new file mode 100644 index 0000000000..954acda5c2 --- /dev/null +++ b/pageserver/src/layered_repository/filename.rs @@ -0,0 +1,172 @@ +use crate::layered_repository::storage_layer::{SegmentTag}; +use crate::relish::*; +use crate::PageServerConf; +use crate::{ZTenantId, ZTimelineId}; +use std::fmt; +use std::fs; + +use anyhow::Result; +use log::*; +use zenith_utils::lsn::Lsn; + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] +pub struct SnapshotFileName { + pub seg: SegmentTag, + pub start_lsn: Lsn, + pub end_lsn: Lsn, + pub dropped: bool, +} + +impl SnapshotFileName { + fn from_str(fname: &str) -> Option { + // Split the filename into parts + // + // ______ + // + // or if it was dropped: + // + // _______DROPPED + // + let rel; + let mut parts; + if let Some(rest) = fname.strip_prefix("rel_") { + parts = rest.split('_'); + rel = RelishTag::Relation(RelTag { + spcnode: parts.next()?.parse::().ok()?, + dbnode: parts.next()?.parse::().ok()?, + relnode: parts.next()?.parse::().ok()?, + forknum: parts.next()?.parse::().ok()?, + }); + } else if let Some(rest) = fname.strip_prefix("pg_xact_") { + parts = rest.split('_'); + rel = RelishTag::Slru { + slru: SlruKind::Clog, + segno: u32::from_str_radix(parts.next()?, 16).ok()?, + }; + } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") { + parts = rest.split('_'); + rel = RelishTag::Slru { + slru: SlruKind::MultiXactMembers, + segno: u32::from_str_radix(parts.next()?, 16).ok()?, + }; + } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") { + parts = rest.split('_'); + rel = RelishTag::Slru { + slru: SlruKind::MultiXactOffsets, + segno: u32::from_str_radix(parts.next()?, 16).ok()?, + }; + } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") { + parts = rest.split('_'); + rel = RelishTag::FileNodeMap { + spcnode: parts.next()?.parse::().ok()?, + dbnode: parts.next()?.parse::().ok()?, + }; + } else if let Some(rest) = fname.strip_prefix("pg_twophase_") { + parts = rest.split('_'); + rel = RelishTag::TwoPhase { + xid: parts.next()?.parse::().ok()?, + }; + } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") { + parts = rest.split('_'); + rel = RelishTag::Checkpoint; + } else if let Some(rest) = fname.strip_prefix("pg_control_") { + parts = rest.split('_'); + rel = RelishTag::ControlFile; + } else { + return None; + } + + let segno = parts.next()?.parse::().ok()?; + + let seg = SegmentTag { rel, segno }; + + let start_lsn = Lsn::from_hex(parts.next()?).ok()?; + let end_lsn = Lsn::from_hex(parts.next()?).ok()?; + + let mut dropped = false; + if let Some(suffix) = parts.next() { + if suffix == "DROPPED" { + dropped = true; + } else { + warn!("unrecognized filename in timeline dir: {}", fname); + return None; + } + } + if parts.next().is_some() { + warn!("unrecognized filename in timeline dir: {}", fname); + return None; + } + + Some(SnapshotFileName { + seg, + start_lsn, + end_lsn, + dropped, + }) + } + + fn to_string(&self) -> String { + let basename = match self.seg.rel { + RelishTag::Relation(reltag) => format!( + "rel_{}_{}_{}_{}", + reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum + ), + RelishTag::Slru { + slru: SlruKind::Clog, + segno, + } => format!("pg_xact_{:04X}", segno), + RelishTag::Slru { + slru: SlruKind::MultiXactMembers, + segno, + } => format!("pg_multixact_members_{:04X}", segno), + RelishTag::Slru { + slru: SlruKind::MultiXactOffsets, + segno, + } => format!("pg_multixact_offsets_{:04X}", segno), + RelishTag::FileNodeMap { spcnode, dbnode } => { + format!("pg_filenodemap_{}_{}", spcnode, dbnode) + } + RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid), + RelishTag::Checkpoint => format!("pg_control_checkpoint"), + RelishTag::ControlFile => format!("pg_control"), + }; + + format!( + "{}_{}_{:016X}_{:016X}{}", + basename, + self.seg.segno, + u64::from(self.start_lsn), + u64::from(self.end_lsn), + if self.dropped { "_DROPPED" } else { "" } + ) + } +} + +impl fmt::Display for SnapshotFileName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.to_string()) + } +} + + +/// Create SnapshotFileName structs representing all files on disk +/// +// TODO: returning an Iterator would be more idiomatic +pub fn list_snapshot_files( + conf: &'static PageServerConf, + timelineid: ZTimelineId, + tenantid: ZTenantId, +) -> Result> { + let path = conf.timeline_path(&timelineid, &tenantid); + + let mut snapfiles: Vec = Vec::new(); + for direntry in fs::read_dir(path)? { + let fname = direntry?.file_name(); + let fname = fname.to_str().unwrap(); + + if let Some(snapfilename) = SnapshotFileName::from_str(fname) { + snapfiles.push(snapfilename); + } + } + return Ok(snapfiles); +} diff --git a/pageserver/src/layered_repository/snapshot_layer.rs b/pageserver/src/layered_repository/snapshot_layer.rs index e0f4e77995..084d023cbe 100644 --- a/pageserver/src/layered_repository/snapshot_layer.rs +++ b/pageserver/src/layered_repository/snapshot_layer.rs @@ -40,19 +40,18 @@ use crate::layered_repository::storage_layer::{ Layer, PageReconstructData, PageVersion, SegmentTag, }; -use crate::relish::*; +use crate::layered_repository::filename::{SnapshotFileName}; use crate::PageServerConf; use crate::{ZTenantId, ZTimelineId}; use anyhow::{bail, Result}; use log::*; use std::collections::BTreeMap; -use std::fmt; use std::fs; use std::fs::File; use std::io::Write; use std::ops::Bound::Included; use std::path::PathBuf; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Mutex, MutexGuard}; use bookfile::{Book, BookWriter}; @@ -65,145 +64,6 @@ static SNAPSHOT_FILE_MAGIC: u32 = 0x5A616E01; static PAGE_VERSIONS_CHAPTER: u64 = 1; static REL_SIZES_CHAPTER: u64 = 2; -#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone)] -struct SnapshotFileName { - seg: SegmentTag, - start_lsn: Lsn, - end_lsn: Lsn, - dropped: bool, -} - -impl SnapshotFileName { - fn from_str(fname: &str) -> Option { - // Split the filename into parts - // - // ______ - // - // or if it was dropped: - // - // _______DROPPED - // - let rel; - let mut parts; - if let Some(rest) = fname.strip_prefix("rel_") { - parts = rest.split('_'); - rel = RelishTag::Relation(RelTag { - spcnode: parts.next()?.parse::().ok()?, - dbnode: parts.next()?.parse::().ok()?, - relnode: parts.next()?.parse::().ok()?, - forknum: parts.next()?.parse::().ok()?, - }); - } else if let Some(rest) = fname.strip_prefix("pg_xact_") { - parts = rest.split('_'); - rel = RelishTag::Slru { - slru: SlruKind::Clog, - segno: u32::from_str_radix(parts.next()?, 16).ok()?, - }; - } else if let Some(rest) = fname.strip_prefix("pg_multixact_members_") { - parts = rest.split('_'); - rel = RelishTag::Slru { - slru: SlruKind::MultiXactMembers, - segno: u32::from_str_radix(parts.next()?, 16).ok()?, - }; - } else if let Some(rest) = fname.strip_prefix("pg_multixact_offsets_") { - parts = rest.split('_'); - rel = RelishTag::Slru { - slru: SlruKind::MultiXactOffsets, - segno: u32::from_str_radix(parts.next()?, 16).ok()?, - }; - } else if let Some(rest) = fname.strip_prefix("pg_filenodemap_") { - parts = rest.split('_'); - rel = RelishTag::FileNodeMap { - spcnode: parts.next()?.parse::().ok()?, - dbnode: parts.next()?.parse::().ok()?, - }; - } else if let Some(rest) = fname.strip_prefix("pg_twophase_") { - parts = rest.split('_'); - rel = RelishTag::TwoPhase { - xid: parts.next()?.parse::().ok()?, - }; - } else if let Some(rest) = fname.strip_prefix("pg_control_checkpoint_") { - parts = rest.split('_'); - rel = RelishTag::Checkpoint; - } else if let Some(rest) = fname.strip_prefix("pg_control_") { - parts = rest.split('_'); - rel = RelishTag::ControlFile; - } else { - return None; - } - - let segno = parts.next()?.parse::().ok()?; - - let seg = SegmentTag { rel, segno }; - - let start_lsn = Lsn::from_hex(parts.next()?).ok()?; - let end_lsn = Lsn::from_hex(parts.next()?).ok()?; - - let mut dropped = false; - if let Some(suffix) = parts.next() { - if suffix == "DROPPED" { - dropped = true; - } else { - warn!("unrecognized filename in timeline dir: {}", fname); - return None; - } - } - if parts.next().is_some() { - warn!("unrecognized filename in timeline dir: {}", fname); - return None; - } - - Some(SnapshotFileName { - seg, - start_lsn, - end_lsn, - dropped, - }) - } - - fn to_string(&self) -> String { - let basename = match self.seg.rel { - RelishTag::Relation(reltag) => format!( - "rel_{}_{}_{}_{}", - reltag.spcnode, reltag.dbnode, reltag.relnode, reltag.forknum - ), - RelishTag::Slru { - slru: SlruKind::Clog, - segno, - } => format!("pg_xact_{:04X}", segno), - RelishTag::Slru { - slru: SlruKind::MultiXactMembers, - segno, - } => format!("pg_multixact_members_{:04X}", segno), - RelishTag::Slru { - slru: SlruKind::MultiXactOffsets, - segno, - } => format!("pg_multixact_offsets_{:04X}", segno), - RelishTag::FileNodeMap { spcnode, dbnode } => { - format!("pg_filenodemap_{}_{}", spcnode, dbnode) - } - RelishTag::TwoPhase { xid } => format!("pg_twophase_{}", xid), - RelishTag::Checkpoint => format!("pg_control_checkpoint"), - RelishTag::ControlFile => format!("pg_control"), - }; - - format!( - "{}_{}_{:016X}_{:016X}{}", - basename, - self.seg.segno, - u64::from(self.start_lsn), - u64::from(self.end_lsn), - if self.dropped { "_DROPPED" } else { "" } - ) - } -} - -impl fmt::Display for SnapshotFileName { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.to_string()) - } -} - /// /// SnapshotLayer is the in-memory data structure associated with an /// on-disk snapshot file. We keep a SnapshotLayer in memory for each @@ -474,38 +334,28 @@ impl SnapshotLayer { /// Create SnapshotLayers representing all files on disk /// // TODO: returning an Iterator would be more idiomatic - pub fn list_snapshot_files( + pub fn load_snapshot_layer( conf: &'static PageServerConf, timelineid: ZTimelineId, tenantid: ZTenantId, - ) -> Result>> { - let path = conf.timeline_path(&timelineid, &tenantid); + filename: &SnapshotFileName, + ) -> Result { + let snapfile = SnapshotLayer { + conf, + timelineid, + tenantid, + seg: filename.seg, + start_lsn: filename.start_lsn, + end_lsn: filename.end_lsn, + dropped: filename.dropped, + inner: Mutex::new(SnapshotLayerInner { + loaded: false, + page_versions: BTreeMap::new(), + relsizes: BTreeMap::new(), + }), + }; - let mut snapfiles: Vec> = Vec::new(); - for direntry in fs::read_dir(path)? { - let fname = direntry?.file_name(); - let fname = fname.to_str().unwrap(); - - if let Some(snapfilename) = SnapshotFileName::from_str(fname) { - let snapfile = SnapshotLayer { - conf, - timelineid, - tenantid, - seg: snapfilename.seg, - start_lsn: snapfilename.start_lsn, - end_lsn: snapfilename.end_lsn, - dropped: snapfilename.dropped, - inner: Mutex::new(SnapshotLayerInner { - loaded: false, - page_versions: BTreeMap::new(), - relsizes: BTreeMap::new(), - }), - }; - - snapfiles.push(Arc::new(snapfile)); - } - } - return Ok(snapfiles); + Ok(snapfile) } pub fn delete(&self) -> Result<()> {