rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-08 14:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add 'dump_layerfile' utility for debugging.

Browse Source 
Seems handy for getting a quick idea of what's stored in an
image or delta layer file.

Example output on a file after runnnig pgbench for a while:

    % ./target/debug/dump_layerfile pgbench_layers/pg_control_checkpoint_0_00000000016B914A
    ----- image layer for checkpoint.0 at 0/16B914A ----
    non-blocky (88 bytes)
    % ./target/debug/dump_layerfile pgbench_layers/pg_xact_0000_0_000000000412FD40
    ----- image layer for pg_xact/0000.0 at 0/412FD40 ----
    (1) blocks
    % ./target/debug/dump_layerfile pgbench_layers/rel_1663_14236_1247_0_0_00000000016B914A_000000000412FD40 | head -n 20
    ----- delta layer for 1663/14236/1247.0 0/16B914A-0/412FD40 ----
    --- relsizes ---
      0/16B914A: 14
      0/16CA559: 15
    --- page versions ---
      blk 13 at 0/16BB1D2:  rec 8162 bytes will_init: true HEAP INSERT
      blk 14 at 0/16CA559:  rec 8241 bytes will_init: true XLOG FPI
      blk 14 at 0/16CA637:  rec 215 bytes will_init: true HEAP INSERT
      blk 14 at 0/16DF14F:  rec 215 bytes will_init: false HEAP INSERT
      blk 14 at 0/16DF3A7:  rec 215 bytes will_init: false HEAP INSERT
      blk 14 at 0/16E0637:  rec 215 bytes will_init: false HEAP INSERT
      blk 14 at 0/16E088F:  rec 215 bytes will_init: false HEAP INSERT
      blk 14 at 0/16E5F9F:  rec 215 bytes will_init: false HEAP INSERT
      blk 14 at 0/16E620F:  rec 215 bytes will_init: false HEAP INSERT
This commit is contained in:
Heikki Linnakangas
2021-08-27 21:10:17 +03:00
committed by Patrick Insinger
parent 5ac3cb1c72
commit d7bebd8074
9 changed files with 295 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
pageserver/src/bin/dump_layerfile.rs Normal file
View File

@@ -0,0 +1,25 @@
//! Main entry point for the dump_layerfile executable
//!
//! A handy tool for debugging, that's all.
use anyhow::Result;
use clap::{App, Arg};
use pageserver::layered_repository::dump_layerfile_from_path;
use std::path::PathBuf;
fn main() -> Result<()> {
let arg_matches = App::new("Zenith dump_layerfile utility")
.about("Dump contents of one layer file, for debugging")
.arg(
Arg::with_name("path")
.help("Path to file to dump")
.required(true)
.index(1),
)
.get_matches();
let path = PathBuf::from(arg_matches.value_of("path").unwrap());
dump_layerfile_from_path(&path)?;
Ok(())
}

22
pageserver/src/layered_repository.rs
View File

@@ -23,6 +23,8 @@ use std::fs;
use std::fs::File;
use std::io::Write;
use std::ops::Bound::Included;
use std::path::Path;
use std::str::FromStr;
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
@@ -49,6 +51,8 @@ mod storage_layer;
use delta_layer::DeltaLayer;
use image_layer::ImageLayer;
use filename::{DeltaFileName, ImageFileName};
use inmemory_layer::InMemoryLayer;
use layer_map::LayerMap;
use storage_layer::{Layer, PageReconstructData, SegmentTag, RELISH_SEG_SIZE};
@@ -1466,3 +1470,21 @@ impl LayeredTimeline {
}
}
}
/// Dump contents of a layer file to stdout.
pub fn dump_layerfile_from_path(path: &Path) -> Result<()> {
let fname = path.file_name().unwrap().to_str().unwrap();
let dummy_tenantid = ZTenantId::from_str("00000000000000000000000000000000")?;
let dummy_timelineid = ZTimelineId::from_str("00000000000000000000000000000000")?;
if let Some(deltafilename) = DeltaFileName::from_str(fname) {
DeltaLayer::new_for_path(path, dummy_timelineid, dummy_tenantid, &deltafilename).dump()?;
} else if let Some(imgfilename) = ImageFileName::from_str(fname) {
ImageLayer::new_for_path(path, dummy_timelineid, dummy_tenantid, &imgfilename).dump()?;
} else {
bail!("unrecognized layer file name : {}", fname);
}
Ok(())
}

108
pageserver/src/layered_repository/delta_layer.rs
View File

@@ -38,11 +38,12 @@
//! parts: the page versions and the relation sizes. They are stored as separate chapters.
//!
use crate::layered_repository::blob::BlobWriter;
use crate::layered_repository::filename::DeltaFileName;
use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
use crate::layered_repository::storage_layer::{
Layer, PageReconstructData, PageVersion, SegmentTag,
};
use crate::repository::WALRecord;
use crate::waldecoder;
use crate::PageServerConf;
use crate::{ZTenantId, ZTimelineId};
use anyhow::{bail, Result};
@@ -50,11 +51,14 @@ use bytes::Bytes;
use log::*;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
// avoid binding to Write (conflicts with std::io::Write)
// while being able to use std::fmt::Write's methods
use std::fmt::Write as _;
use std::fs;
use std::fs::File;
use std::io::Write;
use std::ops::Bound::Included;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex, MutexGuard};
use bookfile::{Book, BookWriter};
@@ -90,7 +94,8 @@ struct PageVersionMeta {
/// be loaded before using it in queries.
///
pub struct DeltaLayer {
conf: &'static PageServerConf,
path_or_conf: PathOrConf,
pub tenantid: ZTenantId,
pub timelineid: ZTimelineId,
pub seg: SegmentTag,
@@ -275,12 +280,52 @@ impl Layer for DeltaLayer {
fn is_incremental(&self) -> bool {
true
}
/// debugging function to print out the contents of the layer
fn dump(&self) -> Result<()> {
println!(
"----- delta layer for {} {}-{} ----",
self.seg, self.start_lsn, self.end_lsn
);
println!("--- relsizes ---");
let inner = self.load()?;
for (k, v) in inner.relsizes.iter() {
println!(" {}: {}", k, v);
}
println!("--- page versions ---");
let (_path, book) = self.open_book()?;
let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
for (k, v) in inner.page_version_metas.iter() {
let mut desc = String::new();
if let Some(page_image_range) = v.page_image_range.as_ref() {
let image = read_blob(&chapter, &page_image_range)?;
write!(&mut desc, " img {} bytes", image.len())?;
}
if let Some(record_range) = v.record_range.as_ref() {
let record_bytes = read_blob(&chapter, record_range)?;
let rec = WALRecord::des(&record_bytes)?;
let wal_desc = waldecoder::describe_wal_record(&rec.rec);
write!(
&mut desc,
" rec {} bytes will_init: {} {}",
rec.rec.len(),
rec.will_init,
wal_desc
)?;
}
println!(" blk {} at {}: {}", k.0, k.1, desc);
}
Ok(())
}
}
impl DeltaLayer {
fn path(&self) -> PathBuf {
Self::path_for(
self.conf,
&self.path_or_conf,
self.timelineid,
self.tenantid,
&DeltaFileName {
@@ -293,13 +338,17 @@ impl DeltaLayer {
}
fn path_for(
conf: &'static PageServerConf,
path_or_conf: &PathOrConf,
timelineid: ZTimelineId,
tenantid: ZTenantId,
fname: &DeltaFileName,
) -> PathBuf {
conf.timeline_path(&timelineid, &tenantid)
.join(fname.to_string())
match path_or_conf {
PathOrConf::Path(path) => path.clone(),
PathOrConf::Conf(conf) => conf
.timeline_path(&timelineid, &tenantid)
.join(fname.to_string()),
}
}
/// Create a new delta file, using the given btreemaps containing the page versions and
@@ -321,7 +370,7 @@ impl DeltaLayer {
relsizes: BTreeMap<Lsn, u32>,
) -> Result<DeltaLayer> {
let delta_layer = DeltaLayer {
conf: conf,
path_or_conf: PathOrConf::Conf(conf),
timelineid: timelineid,
tenantid: tenantid,
seg: seg,
@@ -397,7 +446,7 @@ impl DeltaLayer {
fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
let path = Self::path_for(
self.conf,
&self.path_or_conf,
self.timelineid,
self.tenantid,
&DeltaFileName {
@@ -453,7 +502,7 @@ impl DeltaLayer {
predecessor: Option<Arc<dyn Layer>>,
) -> DeltaLayer {
DeltaLayer {
conf,
path_or_conf: PathOrConf::Conf(conf),
timelineid,
tenantid,
seg: filename.seg,
@@ -469,22 +518,29 @@ impl DeltaLayer {
}
}
/// debugging function to print out the contents of the layer
#[allow(unused)]
pub fn dump(&self) -> String {
let mut result = format!(
"----- snapshot layer for {} {}-{} ----\n",
self.seg, self.start_lsn, self.end_lsn
);
let inner = self.inner.lock().unwrap();
for (k, v) in inner.relsizes.iter() {
result += &format!("{}: {}\n", k, v);
/// Create a DeltaLayer struct representing an existing file on disk.
///
/// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
pub fn new_for_path(
path: &Path,
timelineid: ZTimelineId,
tenantid: ZTenantId,
filename: &DeltaFileName,
) -> DeltaLayer {
DeltaLayer {
path_or_conf: PathOrConf::Path(path.to_path_buf()),
timelineid,
tenantid,
seg: filename.seg,
start_lsn: filename.start_lsn,
end_lsn: filename.end_lsn,
dropped: filename.dropped,
inner: Mutex::new(DeltaLayerInner {
loaded: false,
page_version_metas: BTreeMap::new(),
relsizes: BTreeMap::new(),
}),
predecessor: None,
}
//for (k, v) in inner.page_versions.iter() {
// result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
//}
result
}
}

18
pageserver/src/layered_repository/filename.rs
View File

@@ -7,6 +7,7 @@ use crate::PageServerConf;
use crate::{ZTenantId, ZTimelineId};
use std::fmt;
use std::fs;
use std::path::PathBuf;
use anyhow::Result;
use log::*;
@@ -34,7 +35,7 @@ impl DeltaFileName {
/// Parse a string as a delta file name. Returns None if the filename does not
/// match the expected pattern.
///
fn from_str(fname: &str) -> Option<Self> {
pub fn from_str(fname: &str) -> Option<Self> {
let rel;
let mut parts;
if let Some(rest) = fname.strip_prefix("rel_") {
@@ -170,7 +171,7 @@ impl ImageFileName {
/// Parse a string as an image file name. Returns None if the filename does not
/// match the expected pattern.
///
fn from_str(fname: &str) -> Option<Self> {
pub fn from_str(fname: &str) -> Option<Self> {
let rel;
let mut parts;
if let Some(rest) = fname.strip_prefix("rel_") {
@@ -303,3 +304,16 @@ pub fn list_files(
}
return Ok((imgfiles, deltafiles));
}
/// Helper enum to hold a PageServerConf, or a path
///
/// This is used by DeltaLayer and ImageLayer. Normally, this holds a reference to the
/// global config, and paths to layer files are constructed using the tenant/timeline
/// path from the config. But in the 'dump_layerfile' binary, we need to construct a Layer
/// struct for a file on disk, without having a page server running, so that we have no
/// config. In that case, we use the Path variant to hold the full path to the file on
/// disk.
pub enum PathOrConf {
Path(PathBuf),
Conf(&'static PageServerConf),
}

74
pageserver/src/layered_repository/image_layer.rs
View File

@@ -20,7 +20,7 @@
//!
//! For non-blocky segments, the image can be found in NONBLOCKY_IMAGE_CHAPTER.
//!
use crate::layered_repository::filename::ImageFileName;
use crate::layered_repository::filename::{ImageFileName, PathOrConf};
use crate::layered_repository::storage_layer::{Layer, PageReconstructData, SegmentTag};
use crate::layered_repository::LayeredTimeline;
use crate::layered_repository::RELISH_SEG_SIZE;
@@ -33,7 +33,7 @@ use std::convert::TryInto;
use std::fs;
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::{Mutex, MutexGuard};
use bookfile::{Book, BookWriter};
@@ -57,7 +57,7 @@ const BLOCK_SIZE: usize = 8192;
/// and it needs to be loaded before using it in queries.
///
pub struct ImageLayer {
conf: &'static PageServerConf,
path_or_conf: PathOrConf,
pub tenantid: ZTenantId,
pub timelineid: ZTimelineId,
pub seg: SegmentTag,
@@ -192,12 +192,30 @@ impl Layer for ImageLayer {
fn is_incremental(&self) -> bool {
false
}
/// debugging function to print out the contents of the layer
fn dump(&self) -> Result<()> {
println!("----- image layer for {} at {} ----", self.seg, self.lsn);
let inner = self.load()?;
match inner.image_type {
ImageType::Blocky { num_blocks } => println!("({}) blocks ", num_blocks),
ImageType::NonBlocky => {
let (_path, book) = self.open_book()?;
let chapter = book.read_chapter(NONBLOCKY_IMAGE_CHAPTER)?;
println!("non-blocky ({} bytes)", chapter.len());
}
}
Ok(())
}
}
impl ImageLayer {
fn path(&self) -> PathBuf {
Self::path_for(
self.conf,
&self.path_or_conf,
self.timelineid,
self.tenantid,
&ImageFileName {
@@ -208,13 +226,17 @@ impl ImageLayer {
}
fn path_for(
conf: &'static PageServerConf,
path_or_conf: &PathOrConf,
timelineid: ZTimelineId,
tenantid: ZTenantId,
fname: &ImageFileName,
) -> PathBuf {
conf.timeline_path(&timelineid, &tenantid)
.join(fname.to_string())
match path_or_conf {
PathOrConf::Path(path) => path.to_path_buf(),
PathOrConf::Conf(conf) => conf
.timeline_path(&timelineid, &tenantid)
.join(fname.to_string()),
}
}
/// Create a new image file, using the given array of pages.
@@ -235,7 +257,7 @@ impl ImageLayer {
};
let layer = ImageLayer {
conf: conf,
path_or_conf: PathOrConf::Conf(conf),
timelineid: timelineid,
tenantid: tenantid,
seg: seg,
@@ -354,7 +376,7 @@ impl ImageLayer {
fn open_book(&self) -> Result<(PathBuf, Book<File>)> {
let path = Self::path_for(
self.conf,
&self.path_or_conf,
self.timelineid,
self.tenantid,
&ImageFileName {
@@ -377,7 +399,7 @@ impl ImageLayer {
filename: &ImageFileName,
) -> ImageLayer {
ImageLayer {
conf,
path_or_conf: PathOrConf::Conf(conf),
timelineid,
tenantid,
seg: filename.seg,
@@ -389,17 +411,25 @@ impl ImageLayer {
}
}
/// debugging function to print out the contents of the layer
#[allow(unused)]
pub fn dump(&self) -> String {
let mut result = format!("----- image layer for {} at {} ----\n", self.seg, self.lsn);
//let inner = self.inner.lock().unwrap();
//for (k, v) in inner.page_versions.iter() {
// result += &format!("blk {} at {}: {}/{}\n", k.0, k.1, v.page_image.is_some(), v.record.is_some());
//}
result
/// Create an ImageLayer struct representing an existing file on disk.
///
/// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
pub fn new_for_path(
path: &Path,
timelineid: ZTimelineId,
tenantid: ZTenantId,
filename: &ImageFileName,
) -> ImageLayer {
ImageLayer {
path_or_conf: PathOrConf::Path(path.to_path_buf()),
timelineid,
tenantid,
seg: filename.seg,
lsn: filename.lsn,
inner: Mutex::new(ImageLayerInner {
loaded: false,
image_type: ImageType::Blocky { num_blocks: 0 },
}),
}
}
}

25
pageserver/src/layered_repository/inmemory_layer.rs
View File

@@ -227,6 +227,31 @@ impl Layer for InMemoryLayer {
fn is_incremental(&self) -> bool {
self.img_layer.is_some()
}
/// debugging function to print out the contents of the layer
fn dump(&self) -> Result<()> {
let inner = self.inner.lock().unwrap();
let end_str = inner
.drop_lsn
.as_ref()
.map(|drop_lsn| drop_lsn.to_string())
.unwrap_or_default();
println!(
"----- in-memory layer for {} {}-{} ----",
self.seg, self.start_lsn, end_str
);
for (k, v) in inner.segsizes.iter() {
println!("{}: {}", k, v);
}
//for (k, v) in inner.page_versions.iter() {
// println!("blk {} at {}: {}/{}", k.0, k.1, v.page_image.is_some(), v.record.is_some());
//}
Ok(())
}
}
impl InMemoryLayer {

3
pageserver/src/layered_repository/storage_layer.rs
View File

@@ -145,4 +145,7 @@ pub trait Layer: Send + Sync {
/// Permanently remove this layer from disk.
fn delete(&self) -> Result<()>;
/// Dump summary of the contents of the layer to stdout
fn dump(&self) -> Result<()>;
}

68
pageserver/src/waldecoder.rs
View File

@@ -965,3 +965,71 @@ pub fn decode_wal_record(record: Bytes) -> DecodedWALRecord {
main_data_offset,
}
}
///
/// Build a human-readable string to describe a WAL record
///
/// For debugging purposes
pub fn describe_wal_record(record: &Bytes) -> String {
// TODO: It would be nice to use the PostgreSQL rmgrdesc infrastructure for this.
// Maybe use the postgres wal redo process, the same used for replaying WAL records?
// Or could we compile the rmgrdesc routines into the dump_layer_file() binary directly,
// without worrying about security?
//
// But for now, we have a hand-written code for a few common WAL record types here.
let mut buf = record.clone();
// 1. Parse XLogRecord struct
// FIXME: assume little-endian here
let xlogrec = XLogRecord::from_bytes(&mut buf);
let unknown_str: String;
let result: &str = match xlogrec.xl_rmid {
pg_constants::RM_HEAP2_ID => {
let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
match info {
pg_constants::XLOG_HEAP2_MULTI_INSERT => "HEAP2 MULTI_INSERT",
pg_constants::XLOG_HEAP2_VISIBLE => "HEAP2 VISIBLE",
_ => {
unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
&unknown_str
}
}
}
pg_constants::RM_HEAP_ID => {
let info = xlogrec.xl_info & pg_constants::XLOG_HEAP_OPMASK;
match info {
pg_constants::XLOG_HEAP_INSERT => "HEAP INSERT",
pg_constants::XLOG_HEAP_DELETE => "HEAP DELETE",
pg_constants::XLOG_HEAP_UPDATE => "HEAP UPDATE",
pg_constants::XLOG_HEAP_HOT_UPDATE => "HEAP HOT_UPDATE",
_ => {
unknown_str = format!("HEAP2 UNKNOWN_0x{:02x}", info);
&unknown_str
}
}
}
pg_constants::RM_XLOG_ID => {
let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
match info {
pg_constants::XLOG_FPI => "XLOG FPI",
pg_constants::XLOG_FPI_FOR_HINT => "XLOG FPI_FOR_HINT",
_ => {
unknown_str = format!("XLOG UNKNOWN_0x{:02x}", info);
&unknown_str
}
}
}
rmid => {
let info = xlogrec.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
unknown_str = format!("UNKNOWN_RM_{} INFO_0x{:02x}", rmid, info);
&unknown_str
}
};
String::from(result)
}

2
postgres_ffi/src/pg_constants.rs
View File

@@ -87,6 +87,8 @@ pub const XACT_XINFO_HAS_TWOPHASE: u32 = 1u32 << 4;
pub const XLOG_NEXTOID: u8 = 0x30;
pub const XLOG_SWITCH: u8 = 0x40;
pub const XLOG_SMGR_TRUNCATE: u8 = 0x20;
pub const XLOG_FPI_FOR_HINT: u8 = 0xA0;
pub const XLOG_FPI: u8 = 0xB0;
pub const DB_SHUTDOWNED: u32 = 1;
// From multixact.h