From 96161c8cfd979a3bebebcbcc1f91ff21fe8ad5cb Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Fri, 25 Aug 2023 13:01:42 +0300 Subject: [PATCH] restore Layer::dump --- pageserver/src/tenant.rs | 14 +-- pageserver/src/tenant/layer_map.rs | 4 +- .../src/tenant/storage_layer/delta_layer.rs | 112 +++++++++--------- .../src/tenant/storage_layer/image_layer.rs | 42 +++---- pageserver/src/tenant/storage_layer/layer.rs | 24 ++++ .../src/tenant/storage_layer/layer_desc.rs | 43 ++++--- 6 files changed, 133 insertions(+), 106 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 37c5b8aa4d..a5ff71e626 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -4037,8 +4037,7 @@ mod tests { Ok(()) } - /// FIXME: I don't want to add dump to LayerE, it should be in the ctl - /*#[tokio::test] + #[tokio::test] async fn delta_layer_dumping() -> anyhow::Result<()> { use storage_layer::AsLayerDesc; let (tenant, ctx) = TenantHarness::create("test_layer_dumping")?.load().await; @@ -4048,22 +4047,23 @@ mod tests { make_some_layers(tline.as_ref(), Lsn(0x20)).await?; let layer_map = tline.layers.read().await; - let level0_deltas = layer_map.layer_map().get_level0_deltas()?; + let level0_deltas = layer_map + .layer_map() + .get_level0_deltas()? + .into_iter() + .map(|desc| layer_map.get_from_desc(&desc)) + .collect::>(); assert!(!level0_deltas.is_empty()); for delta in level0_deltas { - let delta = layer_map.get_from_desc(&delta); // Ensure we are dumping a delta layer here assert!(delta.layer_desc().is_delta); - - delta.dump(false, &ctx).await.unwrap(); delta.dump(true, &ctx).await.unwrap(); } Ok(()) } - */ #[tokio::test] async fn corrupt_metadata() -> anyhow::Result<()> { diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index 23d753f535..9b6225501f 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -639,8 +639,8 @@ impl LayerMap { } println!("historic_layers:"); - for layer in self.iter_historic_layers() { - layer.dump(verbose, ctx)?; + for desc in self.iter_historic_layers() { + desc.dump(); } println!("End dump LayerMap"); Ok(()) diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 42bd8fb67f..0418155f16 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -242,16 +242,7 @@ impl AsLayerDesc for DeltaLayer { impl DeltaLayer { pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { - println!( - "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} size {} ----", - self.desc.tenant_id, - self.desc.timeline_id, - self.desc.key_range.start, - self.desc.key_range.end, - self.desc.lsn_range.start, - self.desc.lsn_range.end, - self.desc.file_size, - ); + self.desc.dump(); if !verbose { return Ok(()); @@ -259,55 +250,7 @@ impl DeltaLayer { let inner = self.load(LayerAccessKind::Dump, ctx).await?; - println!( - "index_start_blk: {}, root {}", - inner.index_start_blk, inner.index_root_blk - ); - - let file = &inner.file; - let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( - inner.index_start_blk, - inner.index_root_blk, - file, - ); - - tree_reader.dump().await?; - - let keys = DeltaLayerInner::load_keys(&inner).await?; - - // A subroutine to dump a single blob - async fn dump_blob(val: ValueRef<'_>) -> Result { - let buf = val.reader.read_blob(val.blob_ref.pos()).await?; - let val = Value::des(&buf)?; - let desc = match val { - Value::Image(img) => { - format!(" img {} bytes", img.len()) - } - Value::WalRecord(rec) => { - let wal_desc = walrecord::describe_wal_record(&rec)?; - format!( - " rec {} bytes will_init: {} {}", - buf.len(), - rec.will_init(), - wal_desc - ) - } - }; - Ok(desc) - } - - for entry in keys { - let DeltaEntry { key, lsn, val, .. } = entry; - let desc = match dump_blob(val).await { - Ok(desc) => desc, - Err(err) => { - format!("ERROR: {err}") - } - }; - println!(" key {key} at {lsn}: {desc}"); - } - - Ok(()) + inner.dump().await } fn temp_path_for( @@ -826,6 +769,57 @@ impl DeltaLayerInner { } Ok(all_keys) } + + pub(super) async fn dump(&self) -> anyhow::Result<()> { + println!( + "index_start_blk: {}, root {}", + self.index_start_blk, self.index_root_blk + ); + + let file = &self.file; + let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( + self.index_start_blk, + self.index_root_blk, + file, + ); + + tree_reader.dump().await?; + + let keys = self.load_keys().await?; + + async fn dump_blob(val: ValueRef<'_>) -> anyhow::Result { + let buf = val.reader.read_blob(val.blob_ref.pos()).await?; + let val = Value::des(&buf)?; + let desc = match val { + Value::Image(img) => { + format!(" img {} bytes", img.len()) + } + Value::WalRecord(rec) => { + let wal_desc = walrecord::describe_wal_record(&rec)?; + format!( + " rec {} bytes will_init: {} {}", + buf.len(), + rec.will_init(), + wal_desc + ) + } + }; + Ok(desc) + } + + for entry in keys { + let DeltaEntry { key, lsn, val, .. } = entry; + let desc = match dump_blob(val).await { + Ok(desc) => desc, + Err(err) => { + format!("ERROR: {err}") + } + }; + println!(" key {key} at {lsn}: {desc}"); + } + + Ok(()) + } } /// A set of data associated with a delta layer key and its value diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index acaf473041..54261c95a3 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -169,6 +169,25 @@ impl std::fmt::Debug for ImageLayerInner { } } +impl ImageLayerInner { + pub(super) async fn dump(&self) -> anyhow::Result<()> { + let file = &self.file; + let tree_reader = + DiskBtreeReader::<_, KEY_SIZE>::new(self.index_start_blk, self.index_root_blk, file); + + tree_reader.dump().await?; + + tree_reader + .visit(&[0u8; KEY_SIZE], VisitDirection::Forwards, |key, value| { + println!("key: {} offset {}", hex::encode(key), value); + true + }) + .await?; + + Ok(()) + } +} + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. impl std::fmt::Display for ImageLayer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -184,34 +203,15 @@ impl AsLayerDesc for ImageLayer { impl ImageLayer { pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { - println!( - "----- image layer for ten {} tli {} key {}-{} at {} is_incremental {} size {} ----", - self.desc.tenant_id, - self.desc.timeline_id, - self.desc.key_range.start, - self.desc.key_range.end, - self.lsn, - self.desc.is_incremental(), - self.desc.file_size - ); + self.desc.dump(); if !verbose { return Ok(()); } let inner = self.load(LayerAccessKind::Dump, ctx).await?; - let file = &inner.file; - let tree_reader = - DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file); - tree_reader.dump().await?; - - tree_reader - .visit(&[0u8; KEY_SIZE], VisitDirection::Forwards, |key, value| { - println!("key: {} offset {}", hex::encode(key), value); - true - }) - .await?; + inner.dump().await?; Ok(()) } diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 14c3b587de..854f4859dc 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -247,6 +247,20 @@ impl Layer { pub(crate) fn local_path(&self) -> &Path { &self.0.path } + + /// Traditional debug dumping facility + #[allow(unused)] + pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> anyhow::Result<()> { + self.0.desc.dump(); + + if verbose { + // for now, unconditionally download everything, even if that might not be wanted. + let l = self.0.get_or_maybe_download(true, Some(ctx)).await?; + l.dump(&self.0).await? + } + + Ok(()) + } } /// The download-ness ([`DownloadedLayer`]) can be either resident or wanted evicted. @@ -1102,6 +1116,16 @@ impl DownloadedLayer { Image(i) => i.get_value_reconstruct_data(key, reconstruct_data).await, } } + + async fn dump(&self, owner: &LayerInner) -> anyhow::Result<()> { + use LayerKind::*; + match self.get(owner).await? { + Delta(d) => d.dump().await?, + Image(i) => i.dump().await?, + } + + Ok(()) + } } /// Wrapper around an actual layer implementation. diff --git a/pageserver/src/tenant/storage_layer/layer_desc.rs b/pageserver/src/tenant/storage_layer/layer_desc.rs index ce55b3e018..2e0b0b3e64 100644 --- a/pageserver/src/tenant/storage_layer/layer_desc.rs +++ b/pageserver/src/tenant/storage_layer/layer_desc.rs @@ -1,4 +1,3 @@ -use anyhow::Result; use core::fmt::Display; use std::ops::Range; use utils::{ @@ -6,7 +5,7 @@ use utils::{ lsn::Lsn, }; -use crate::{context::RequestContext, repository::Key}; +use crate::repository::Key; use super::{DeltaFileName, ImageFileName, LayerFileName}; @@ -189,21 +188,31 @@ impl PersistentLayerDesc { self.is_delta } - pub fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { - println!( - "----- layer for ten {} tli {} keys {}-{} lsn {}-{} is_delta {} is_incremental {} size {} ----", - self.tenant_id, - self.timeline_id, - self.key_range.start, - self.key_range.end, - self.lsn_range.start, - self.lsn_range.end, - self.is_delta, - self.is_incremental(), - self.file_size, - ); - - Ok(()) + pub fn dump(&self) { + if self.is_delta { + println!( + "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} is_incremental {} size {} ----", + self.tenant_id, + self.timeline_id, + self.key_range.start, + self.key_range.end, + self.lsn_range.start, + self.lsn_range.end, + self.is_incremental(), + self.file_size, + ); + } else { + println!( + "----- image layer for ten {} tli {} key {}-{} at {} is_incremental {} size {} ----", + self.tenant_id, + self.timeline_id, + self.key_range.start, + self.key_range.end, + self.image_layer_lsn(), + self.is_incremental(), + self.file_size + ); + } } pub fn file_size(&self) -> u64 {