rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-05 20:42:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

pageserver - use VecMap for delta metadata & sizes

Browse Source
This commit is contained in:
Patrick Insinger
2021-10-08 11:28:00 -07:00
committed by Patrick Insinger
parent c4bb6d78d4
commit c356030660
3 changed files with 38 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
pageserver/src/layered_repository/delta_layer.rs
View File

@@ -48,7 +48,7 @@ use crate::{ZTenantId, ZTimelineId};
use anyhow::{bail, ensure, Result};
use log::*;
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use zenith_utils::vec_map::VecMap;
// avoid binding to Write (conflicts with std::io::Write)
// while being able to use std::fmt::Write's methods
use std::fmt::Write as _;
@@ -141,10 +141,10 @@ pub struct DeltaLayerInner {
/// All versions of all pages in the file are are kept here.
/// Indexed by block number and LSN.
page_version_metas: BTreeMap<(u32, Lsn), BlobRange>,
page_version_metas: VecMap<(u32, Lsn), BlobRange>,
/// `relsizes` tracks the size of the relation at different points in time.
relsizes: BTreeMap<Lsn, u32>,
relsizes: VecMap<Lsn, u32>,
}
impl Layer for DeltaLayer {
@@ -215,10 +215,12 @@ impl Layer for DeltaLayer {
// Scan the metadata BTreeMap backwards, starting from the given entry.
let minkey = (blknum, Lsn(0));
let maxkey = (blknum, lsn);
let mut iter = inner
let iter = inner
.page_version_metas
.range((Included(&minkey), Included(&maxkey)));
while let Some(((_blknum, _entry_lsn), blob_range)) = iter.next_back() {
.slice_range((Included(&minkey), Included(&maxkey)))
.iter()
.rev();
for ((_blknum, _lsn), blob_range) in iter {
let pv = PageVersion::des(&read_blob(&page_version_reader, blob_range)?)?;
if let Some(img) = pv.page_image {
@@ -262,15 +264,15 @@ impl Layer for DeltaLayer {
// Scan the BTreeMap backwards, starting from the given entry.
let inner = self.load()?;
let mut iter = inner.relsizes.range((Included(&Lsn(0)), Included(&lsn)));
let slice = inner
.relsizes
.slice_range((Included(&Lsn(0)), Included(&lsn)));
let result;
if let Some((_entry_lsn, entry)) = iter.next_back() {
result = *entry;
if let Some((_entry_lsn, entry)) = slice.last() {
Ok(*entry)
} else {
bail!("could not find seg size in delta layer");
Err(anyhow::anyhow!("could not find seg size in delta layer"))
}
Ok(result)
}
/// Does this segment exist at given LSN?
@@ -290,8 +292,8 @@ impl Layer for DeltaLayer {
///
fn unload(&self) -> Result<()> {
let mut inner = self.inner.lock().unwrap();
inner.page_version_metas = BTreeMap::new();
inner.relsizes = BTreeMap::new();
inner.page_version_metas = VecMap::default();
inner.relsizes = VecMap::default();
inner.loaded = false;
Ok(())
}
@@ -317,13 +319,13 @@ impl Layer for DeltaLayer {
println!("--- relsizes ---");
let inner = self.load()?;
for (k, v) in inner.relsizes.iter() {
for (k, v) in inner.relsizes.as_slice() {
println!(" {}: {}", k, v);
}
println!("--- page versions ---");
let (_path, book) = self.open_book()?;
let chapter = book.chapter_reader(PAGE_VERSIONS_CHAPTER)?;
for ((blk, lsn), blob_range) in inner.page_version_metas.iter() {
for ((blk, lsn), blob_range) in inner.page_version_metas.as_slice() {
let mut desc = String::new();
let buf = read_blob(&chapter, blob_range)?;
@@ -381,7 +383,7 @@ impl DeltaLayer {
end_lsn: Lsn,
dropped: bool,
page_versions: impl Iterator<Item = (u32, Lsn, &'a PageVersion)>,
relsizes: BTreeMap<Lsn, u32>,
relsizes: VecMap<Lsn, u32>,
) -> Result<DeltaLayer> {
if seg.rel.is_blocky() {
assert!(!relsizes.is_empty());
@@ -397,7 +399,7 @@ impl DeltaLayer {
dropped,
inner: Mutex::new(DeltaLayerInner {
loaded: true,
page_version_metas: BTreeMap::new(),
page_version_metas: VecMap::default(),
relsizes,
}),
};
@@ -420,22 +422,23 @@ impl DeltaLayer {
let buf = PageVersion::ser(page_version)?;
let blob_range = page_version_writer.write_blob(&buf)?;
let old = inner.page_version_metas.insert((blknum, lsn), blob_range);
assert!(old.is_none());
inner
.page_version_metas
.append((blknum, lsn), blob_range)
.unwrap();
}
let book = page_version_writer.close()?;
// Write out page versions
let mut chapter = book.new_chapter(PAGE_VERSION_METAS_CHAPTER);
let buf = BTreeMap::ser(&inner.page_version_metas)?;
let buf = VecMap::ser(&inner.page_version_metas)?;
chapter.write_all(&buf)?;
let book = chapter.close()?;
// and relsizes to separate chapter
let mut chapter = book.new_chapter(REL_SIZES_CHAPTER);
let buf = BTreeMap::ser(&inner.relsizes)?;
let buf = VecMap::ser(&inner.relsizes)?;
chapter.write_all(&buf)?;
let book = chapter.close()?;
@@ -522,10 +525,10 @@ impl DeltaLayer {
}
let chapter = book.read_chapter(PAGE_VERSION_METAS_CHAPTER)?;
let page_version_metas = BTreeMap::des(&chapter)?;
let page_version_metas = VecMap::des(&chapter)?;
let chapter = book.read_chapter(REL_SIZES_CHAPTER)?;
let relsizes = BTreeMap::des(&chapter)?;
let relsizes = VecMap::des(&chapter)?;
debug!("loaded from {}", &path.display());
@@ -555,8 +558,8 @@ impl DeltaLayer {
dropped: filename.dropped,
inner: Mutex::new(DeltaLayerInner {
loaded: false,
page_version_metas: BTreeMap::new(),
relsizes: BTreeMap::new(),
page_version_metas: VecMap::default(),
relsizes: VecMap::default(),
}),
}
}
@@ -578,8 +581,8 @@ impl DeltaLayer {
dropped: summary.dropped,
inner: Mutex::new(DeltaLayerInner {
loaded: false,
page_version_metas: BTreeMap::new(),
relsizes: BTreeMap::new(),
page_version_metas: VecMap::default(),
relsizes: VecMap::default(),
}),
})
}

11
pageserver/src/layered_repository/inmemory_layer.rs
View File

@@ -16,7 +16,6 @@ use anyhow::{bail, ensure, Result};
use bytes::Bytes;
use log::*;
use std::cmp::min;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::{Arc, RwLock};
use zenith_utils::vec_map::VecMap;
@@ -691,8 +690,6 @@ impl InMemoryLayer {
assert!(!inner.writeable);
if let Some(drop_lsn) = inner.drop_lsn {
let segsizes_map: BTreeMap<Lsn, u32> =
inner.segsizes.as_slice().iter().cloned().collect();
let delta_layer = DeltaLayer::create(
self.conf,
self.timelineid,
@@ -702,7 +699,7 @@ impl InMemoryLayer {
drop_lsn,
true,
inner.page_versions.ordered_page_version_iter(None),
segsizes_map,
inner.segsizes.clone(),
)?;
trace!(
"freeze: created delta layer for dropped segment {} {}-{}",
@@ -715,15 +712,13 @@ impl InMemoryLayer {
let end_lsn = self.end_lsn.unwrap();
let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1));
let before_segsizes: BTreeMap<Lsn, u32> =
before_segsizes.as_slice().iter().cloned().collect();
let mut before_page_versions = inner.page_versions.ordered_page_version_iter(Some(end_lsn));
let mut frozen_layers: Vec<Arc<dyn Layer>> = Vec::new();
if self.start_lsn != end_lsn {
let (before_segsizes, _after_segsizes) = inner.segsizes.split_at(&Lsn(end_lsn.0 + 1));
// Write the page versions before the cutoff to disk.
let delta_layer = DeltaLayer::create(
self.conf,

4
zenith_utils/src/vec_map.rs
View File

@@ -1,9 +1,11 @@
use std::{cmp::Ordering, ops::RangeBounds};
use serde::{Deserialize, Serialize};
/// Ordered map datastructure implemented in a Vec.
/// Append only - can only add keys that are larger than the
/// current max key.
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct VecMap<K, V>(Vec<(K, V)>);
impl<K, V> Default for VecMap<K, V> {