From 61af9bb889a2d9a1569a07c629cb198bdb317fe9 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 6 May 2021 21:57:10 +0300 Subject: [PATCH] Move a few functions that have been copy-pasted around to shared module. --- Cargo.lock | 1 + pageserver/src/basebackup.rs | 74 +--------------------- pageserver/src/restore_local_repo.rs | 76 +---------------------- pageserver/src/restore_s3.rs | 39 +----------- postgres_ffi/Cargo.toml | 1 + postgres_ffi/src/lib.rs | 1 + postgres_ffi/src/relfile_utils.rs | 92 ++++++++++++++++++++++++++++ 7 files changed, 100 insertions(+), 184 deletions(-) create mode 100644 postgres_ffi/src/relfile_utils.rs diff --git a/Cargo.lock b/Cargo.lock index eab498fdb0..f93136edc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1340,6 +1340,7 @@ dependencies = [ "hex", "log", "rand", + "regex", ] [[package]] diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index cf0314dde2..196f824a63 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -1,12 +1,11 @@ use log::*; -use regex::Regex; -use std::fmt; use std::io::Write; use tar::Builder; use walkdir::WalkDir; use crate::ZTimelineId; +use postgres_ffi::relfile_utils::*; use zenith_utils::lsn::Lsn; /// @@ -99,73 +98,6 @@ pub fn send_snapshot_tarball( Ok(()) } -// formats: -// -// _ -// . -// _. - -#[derive(Debug)] -struct FilePathError { - msg: String, -} - -impl FilePathError { - fn new(msg: &str) -> FilePathError { - FilePathError { - msg: msg.to_string(), - } - } -} - -impl From for FilePathError { - fn from(e: core::num::ParseIntError) -> Self { - return FilePathError { - msg: format!("invalid filename: {}", e), - }; - } -} - -impl fmt::Display for FilePathError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "invalid filename") - } -} - -fn forkname_to_forknum(forkname: Option<&str>) -> Result { - match forkname { - // "main" is not in filenames, it's implicit if the fork name is not present - None => Ok(0), - Some("fsm") => Ok(1), - Some("vm") => Ok(2), - Some("init") => Ok(3), - Some(_) => Err(FilePathError::new("invalid forkname")), - } -} - -fn parse_filename(fname: &str) -> Result<(u32, u32, u32), FilePathError> { - let re = Regex::new(r"^(?P\d+)(_(?P[a-z]+))?(\.(?P\d+))?$").unwrap(); - - let caps = re - .captures(fname) - .ok_or_else(|| FilePathError::new("invalid relation data file name"))?; - - let relnode_str = caps.name("relnode").unwrap().as_str(); - let relnode = u32::from_str_radix(relnode_str, 10)?; - - let forkname = caps.name("forkname").map(|f| f.as_str()); - let forknum = forkname_to_forknum(forkname)?; - - let segno_match = caps.name("segno"); - let segno = if segno_match.is_none() { - 0 - } else { - u32::from_str_radix(segno_match.unwrap().as_str(), 10)? - }; - - Ok((relnode, forknum, segno)) -} - /// /// Parse a path, relative to the root of PostgreSQL data directory, as /// a PostgreSQL relation data file. @@ -189,7 +121,7 @@ fn parse_rel_file_path(path: &str) -> Result<(), FilePathError> { * . */ if let Some(fname) = path.strip_prefix("global/") { - let (_relnode, _forknum, _segno) = parse_filename(fname)?; + let (_relnode, _forknum, _segno) = parse_relfilename(fname)?; Ok(()) } else if let Some(dbpath) = path.strip_prefix("base/") { @@ -205,7 +137,7 @@ fn parse_rel_file_path(path: &str) -> Result<(), FilePathError> { return Err(FilePathError::new("invalid relation data file name")); }; - let (_relnode, _forknum, _segno) = parse_filename(fname)?; + let (_relnode, _forknum, _segno) = parse_relfilename(fname)?; Ok(()) } else if let Some(_) = path.strip_prefix("pg_tblspc/") { diff --git a/pageserver/src/restore_local_repo.rs b/pageserver/src/restore_local_repo.rs index 0e0108eb33..39fa610350 100644 --- a/pageserver/src/restore_local_repo.rs +++ b/pageserver/src/restore_local_repo.rs @@ -11,11 +11,8 @@ // use log::*; -use regex::Regex; -use std::fmt; use std::cmp::max; -use std::error::Error; use std::fs; use std::fs::File; use std::io::Read; @@ -31,6 +28,7 @@ use crate::waldecoder::{decode_wal_record, WalStreamDecoder}; use crate::PageServerConf; use crate::ZTimelineId; use postgres_ffi::pg_constants; +use postgres_ffi::relfile_utils::*; use postgres_ffi::xlog_utils::*; use zenith_utils::lsn::Lsn; @@ -370,75 +368,3 @@ fn restore_wal(timeline: &dyn Timeline, timelineid: ZTimelineId, startpoint: Lsn Ok(()) } - -#[derive(Debug, Clone)] -struct FilePathError { - msg: String, -} - -impl Error for FilePathError { - fn description(&self) -> &str { - &self.msg - } -} -impl FilePathError { - fn new(msg: &str) -> FilePathError { - FilePathError { - msg: msg.to_string(), - } - } -} - -impl From for FilePathError { - fn from(e: core::num::ParseIntError) -> Self { - return FilePathError { - msg: format!("invalid filename: {}", e), - }; - } -} - -impl fmt::Display for FilePathError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "invalid filename") - } -} - -fn forkname_to_forknum(forkname: Option<&str>) -> Result { - match forkname { - // "main" is not in filenames, it's implicit if the fork name is not present - None => Ok(0), - Some("fsm") => Ok(1), - Some("vm") => Ok(2), - Some("init") => Ok(3), - Some(_) => Err(FilePathError::new("invalid forkname")), - } -} - -// formats: -// -// _ -// . -// _. - -fn parse_relfilename(fname: &str) -> Result<(u32, u32, u32), FilePathError> { - let re = Regex::new(r"^(?P\d+)(_(?P[a-z]+))?(\.(?P\d+))?$").unwrap(); - - let caps = re - .captures(fname) - .ok_or_else(|| FilePathError::new("invalid relation data file name"))?; - - let relnode_str = caps.name("relnode").unwrap().as_str(); - let relnode = u32::from_str_radix(relnode_str, 10)?; - - let forkname = caps.name("forkname").map(|f| f.as_str()); - let forknum = forkname_to_forknum(forkname)?; - - let segno_match = caps.name("segno"); - let segno = if segno_match.is_none() { - 0 - } else { - u32::from_str_radix(segno_match.unwrap().as_str(), 10)? - }; - - Ok((relnode, forknum, segno)) -} diff --git a/pageserver/src/restore_s3.rs b/pageserver/src/restore_s3.rs index e3c4b2fffc..2996d7650c 100644 --- a/pageserver/src/restore_s3.rs +++ b/pageserver/src/restore_s3.rs @@ -24,6 +24,7 @@ use futures::future; use crate::{page_cache, PageServerConf}; use postgres_ffi::pg_constants; +use postgres_ffi::relfile_utils::*; struct Storage { region: Region, @@ -128,44 +129,6 @@ async fn restore_chunk(conf: &PageServerConf) -> Result<(), S3Error> { Ok(()) } -#[derive(Debug)] -struct FilePathError { - msg: String, -} - -impl FilePathError { - fn new(msg: &str) -> FilePathError { - FilePathError { - msg: msg.to_string(), - } - } -} - -impl From for FilePathError { - fn from(e: core::num::ParseIntError) -> Self { - return FilePathError { - msg: format!("invalid filename: {}", e), - }; - } -} - -impl fmt::Display for FilePathError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "invalid filename") - } -} - -fn forkname_to_forknum(forkname: Option<&str>) -> Result { - match forkname { - // "main" is not in filenames, it's implicit if the fork name is not present - None => Ok(0), - Some("fsm") => Ok(1), - Some("vm") => Ok(2), - Some("init") => Ok(3), - Some(_) => Err(FilePathError::new("invalid forkname")), - } -} - #[derive(Debug)] struct ParsedBaseImageFileName { pub spcnode: u32, diff --git a/postgres_ffi/Cargo.toml b/postgres_ffi/Cargo.toml index 82d498b32c..ca193ee02a 100644 --- a/postgres_ffi/Cargo.toml +++ b/postgres_ffi/Cargo.toml @@ -9,6 +9,7 @@ edition = "2018" [dependencies] chrono = "0.4.19" rand = "0.8.3" +regex = "1.4.5" bytes = "1.0.1" byteorder = "1.4.3" anyhow = "1.0" diff --git a/postgres_ffi/src/lib.rs b/postgres_ffi/src/lib.rs index 8513261caf..a1a12a901d 100644 --- a/postgres_ffi/src/lib.rs +++ b/postgres_ffi/src/lib.rs @@ -4,6 +4,7 @@ include!(concat!(env!("OUT_DIR"), "/bindings.rs")); pub mod pg_constants; +pub mod relfile_utils; pub mod xlog_utils; use bytes::{Buf, Bytes, BytesMut}; diff --git a/postgres_ffi/src/relfile_utils.rs b/postgres_ffi/src/relfile_utils.rs new file mode 100644 index 0000000000..94ee3e74a5 --- /dev/null +++ b/postgres_ffi/src/relfile_utils.rs @@ -0,0 +1,92 @@ +/// +/// Common utilities for dealing with PostgreSQL relation files. +/// +use regex::Regex; +use std::error::Error; +use std::fmt; + +#[derive(Debug, Clone)] +pub struct FilePathError { + msg: String, +} + +impl Error for FilePathError { + fn description(&self) -> &str { + &self.msg + } +} +impl FilePathError { + pub fn new(msg: &str) -> FilePathError { + FilePathError { + msg: msg.to_string(), + } + } +} + +impl From for FilePathError { + fn from(e: core::num::ParseIntError) -> Self { + return FilePathError { + msg: format!("invalid filename: {}", e), + }; + } +} + +impl fmt::Display for FilePathError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "invalid filename") + } +} + +/// Convert Postgres relation file's fork suffix to fork number. +pub fn forkname_to_number(forkname: Option<&str>) -> Result { + match forkname { + // "main" is not in filenames, it's implicit if the fork name is not present + None => Ok(0), + Some("fsm") => Ok(1), + Some("vm") => Ok(2), + Some("init") => Ok(3), + Some(_) => Err(FilePathError::new("invalid forkname")), + } +} + +/// Convert Postgres fork number to the right suffix of the relation data file. +pub fn forknumber_to_name(forknum: u8) -> Option<&'static str> { + match forknum { + 0 => None, + 1 => Some("fsm"), + 2 => Some("vm"), + 3 => Some("init"), + _ => panic!("unrecognized fork number"), + } +} + +/// +/// Parse a filename of a relation file. Returns (relfilenode, forknum, segno) tuple. +/// +/// Formats: +/// +/// _ +/// . +/// _. +pub fn parse_relfilename(fname: &str) -> Result<(u32, u8, u32), FilePathError> { + let re = Regex::new(r"^(?P\d+)(_(?P[a-z]+))?(\.(?P\d+))?$").unwrap(); + + let caps = re + .captures(fname) + .ok_or_else(|| FilePathError::new("invalid relation data file name"))?; + + let relnode_str = caps.name("relnode").unwrap().as_str(); + let relnode = u32::from_str_radix(relnode_str, 10)?; + + let forkname = caps.name("forkname").map(|f| f.as_str()); + let forknum = forkname_to_number(forkname)?; + + let segno_match = caps.name("segno"); + let segno = if segno_match.is_none() { + 0 + } else { + u32::from_str_radix(segno_match.unwrap().as_str(), 10)? + }; + + Ok((relnode, forknum, segno)) +}