From c356030660fbe24e69c3590786a3431b7a9760fc Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Fri, 8 Oct 2021 11:28:00 -0700 Subject: [PATCH] pageserver - use VecMap for delta metadata & sizes --- .../src/layered_repository/delta_layer.rs | 61 ++++++++++--------- .../src/layered_repository/inmemory_layer.rs | 11 +--- zenith_utils/src/vec_map.rs | 4 +- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index 4ea415c899..b96ea88920 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -48,7 +48,7 @@ use crate::{ZTenantId, ZTimelineId}; use anyhow::{bail, ensure, Result}; use log::*; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; +use zenith_utils::vec_map::VecMap; // avoid binding to Write (conflicts with std::io::Write) // while being able to use std::fmt::Write's methods use std::fmt::Write as _; @@ -141,10 +141,10 @@ pub struct DeltaLayerInner { /// All versions of all pages in the file are are kept here. /// Indexed by block number and LSN. - page_version_metas: BTreeMap<(u32, Lsn), BlobRange>, + page_version_metas: VecMap<(u32, Lsn), BlobRange>, /// `relsizes` tracks the size of the relation at different points in time. - relsizes: BTreeMap, + relsizes: VecMap, } impl Layer for DeltaLayer { @@ -215,10 +215,12 @@ impl Layer for DeltaLayer { // Scan the metadata BTreeMap backwards, starting from the given entry. let minkey = (blknum, Lsn(0)); let maxkey = (blknum, lsn); - let mut iter = inner + let iter = inner .page_version_metas - .range((Included(&minkey), Included(&maxkey))); - while let Some(((_blknum, _entry_lsn), blob_range)) = iter.next_back() { + .slice_range((Included(&minkey), Included(&maxkey))) + .iter() + .rev(); + for ((_blknum, _lsn), blob_range) in iter { let pv = PageVersion::des(&read_blob(&page_version_reader, blob_range)?)?; if let Some(img) = pv.page_image { @@ -262,15 +264,15 @@ impl Layer for DeltaLayer { // Scan the BTreeMap backwards, starting from the given entry. let inner = self.load()?; - let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn))); + let slice = inner + .relsizes + .slice_range((Included(&Lsn(0)), Included(&lsn))); - let result; - if let Some((_entry_lsn, entry)) = iter.next_back() { - result = *entry; + if let Some((_entry_lsn, entry)) = slice.last() { + Ok(*entry) } else { - bail!("could not find seg size in delta layer"); + Err(anyhow::anyhow!("could not find seg size in delta layer")) } - Ok(result) } /// Does this segment exist at given LSN? @@ -290,8 +292,8 @@ impl Layer for DeltaLayer { /// fn unload(&self) -> Result<()> { let mut inner = self.inner.lock().unwrap(); - inner.page_version_metas = BTreeMap::new(); - inner.relsizes = BTreeMap::new(); + inner.page_version_metas = VecMap::default(); + inner.relsizes = VecMap::default(); inner.loaded = false; Ok(()) } @@ -317,13 +319,13 @@ impl Layer for DeltaLayer { println!("--- relsizes ---"); let inner = self.load()?; - for (k, v) in inner.relsizes.iter() { + for (k, v) in inner.relsizes.as_slice() { println!(" {}: {}", k, v); } println!("--- page versions ---"); let (_path, book) = self.open_book()?; let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?; - for ((blk, lsn), blob_range) in inner.page_version_metas.iter() { + for ((blk, lsn), blob_range) in inner.page_version_metas.as_slice() { let mut desc = String::new(); let buf = read_blob(&chapter, blob_range)?; @@ -381,7 +383,7 @@ impl DeltaLayer { end_lsn: Lsn, dropped: bool, page_versions: impl Iterator, - relsizes: BTreeMap, + relsizes: VecMap, ) -> Result { if seg.rel.is_blocky() { assert!(!relsizes.is_empty()); @@ -397,7 +399,7 @@ impl DeltaLayer { dropped, inner: Mutex::new(DeltaLayerInner { loaded: true, - page_version_metas: BTreeMap::new(), + page_version_metas: VecMap::default(), relsizes, }), }; @@ -420,22 +422,23 @@ impl DeltaLayer { let buf = PageVersion::ser(page_version)?; let blob_range = page_version_writer.write_blob(&buf)?; - let old = inner.page_version_metas.insert((blknum, lsn), blob_range); - - assert!(old.is_none()); + inner + .page_version_metas + .append((blknum, lsn), blob_range) + .unwrap(); } let book = page_version_writer.close()?; // Write out page versions let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER); - let buf = BTreeMap::ser(&inner.page_version_metas)?; + let buf = VecMap::ser(&inner.page_version_metas)?; chapter.write_all(&buf)?; let book = chapter.close()?; // and relsizes to separate chapter let mut chapter = book.new_chapter(REL_SIZES_CHAPTER); - let buf = BTreeMap::ser(&inner.relsizes)?; + let buf = VecMap::ser(&inner.relsizes)?; chapter.write_all(&buf)?; let book = chapter.close()?; @@ -522,10 +525,10 @@ impl DeltaLayer { } let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?; - let page_version_metas = BTreeMap::des(&chapter)?; + let page_version_metas = VecMap::des(&chapter)?; let chapter = book.read_chapter(REL_SIZES_CHAPTER)?; - let relsizes = BTreeMap::des(&chapter)?; + let relsizes = VecMap::des(&chapter)?; debug!("loaded from {}", &path.display()); @@ -555,8 +558,8 @@ impl DeltaLayer { dropped: filename.dropped, inner: Mutex::new(DeltaLayerInner { loaded: false, - page_version_metas: BTreeMap::new(), - relsizes: BTreeMap::new(), + page_version_metas: VecMap::default(), + relsizes: VecMap::default(), }), } } @@ -578,8 +581,8 @@ impl DeltaLayer { dropped: summary.dropped, inner: Mutex::new(DeltaLayerInner { loaded: false, - page_version_metas: BTreeMap::new(), - relsizes: BTreeMap::new(), + page_version_metas: VecMap::default(), + relsizes: VecMap::default(), }), }) } diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index b492ecd252..5f5307fec0 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -16,7 +16,6 @@ use anyhow::{bail, ensure, Result}; use bytes::Bytes; use log::*; use std::cmp::min; -use std::collections::BTreeMap; use std::path::PathBuf; use std::sync::{Arc, RwLock}; use zenith_utils::vec_map::VecMap; @@ -691,8 +690,6 @@ impl InMemoryLayer { assert!(!inner.writeable); if let Some(drop_lsn) = inner.drop_lsn { - let segsizes_map: BTreeMap = - inner.segsizes.as_slice().iter().cloned().collect(); let delta_layer = DeltaLayer::create( self.conf, self.timelineid, @@ -702,7 +699,7 @@ impl InMemoryLayer { drop_lsn, true, inner.page_versions.ordered_page_version_iter(None), - segsizes_map, + inner.segsizes.clone(), )?; trace!( "freeze: created delta layer for dropped segment {} {}-{}", @@ -715,15 +712,13 @@ impl InMemoryLayer { let end_lsn = self.end_lsn.unwrap(); - let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1)); - let before_segsizes: BTreeMap = - before_segsizes.as_slice().iter().cloned().collect(); - let mut before_page_versions = inner.page_versions.ordered_page_version_iter(Some(end_lsn)); let mut frozen_layers: Vec> = Vec::new(); if self.start_lsn != end_lsn { + let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1)); + // Write the page versions before the cutoff to disk. let delta_layer = DeltaLayer::create( self.conf, diff --git a/zenith_utils/src/vec_map.rs b/zenith_utils/src/vec_map.rs index 5d14245de2..0fd33bf489 100644 --- a/zenith_utils/src/vec_map.rs +++ b/zenith_utils/src/vec_map.rs @@ -1,9 +1,11 @@ use std::{cmp::Ordering, ops::RangeBounds}; +use serde::{Deserialize, Serialize}; + /// Ordered map datastructure implemented in a Vec. /// Append only - can only add keys that are larger than the /// current max key. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct VecMap(Vec<(K, V)>); impl Default for VecMap {