From 98f58fa2abad26a4eea85a8d1d38fa8568b31def Mon Sep 17 00:00:00 2001 From: Patrick Insinger Date: Tue, 28 Sep 2021 22:17:58 -0700 Subject: [PATCH] use for seqsizes in inmemorylayer --- .../src/layered_repository/delta_layer.rs | 5 +-- .../src/layered_repository/inmemory_layer.rs | 37 +++++++++---------- zenith_utils/src/ordered_vec.rs | 32 +++++++++++++++- 3 files changed, 50 insertions(+), 24 deletions(-) diff --git a/pageserver/src/layered_repository/delta_layer.rs b/pageserver/src/layered_repository/delta_layer.rs index 80299b36fa..2ea0548c88 100644 --- a/pageserver/src/layered_repository/delta_layer.rs +++ b/pageserver/src/layered_repository/delta_layer.rs @@ -48,7 +48,6 @@ use crate::{ZTenantId, ZTimelineId}; use anyhow::{bail, Result}; use log::*; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; // avoid binding to Write (conflicts with std::io::Write) // while being able to use std::fmt::Write's methods use std::fmt::Write as _; @@ -393,7 +392,7 @@ impl DeltaLayer { dropped: bool, predecessor: Option>, page_versions: impl Iterator, - relsizes: BTreeMap, + relsizes: OrderedVec, ) -> Result { let delta_layer = DeltaLayer { path_or_conf: PathOrConf::Conf(conf), @@ -406,7 +405,7 @@ impl DeltaLayer { inner: Mutex::new(DeltaLayerInner { loaded: true, page_version_metas: OrderedVec::default(), // TODO create with a size estimate - relsizes: OrderedVec::from(relsizes), + relsizes, }), predecessor, }; diff --git a/pageserver/src/layered_repository/inmemory_layer.rs b/pageserver/src/layered_repository/inmemory_layer.rs index c0fbad7969..1e3d99f713 100644 --- a/pageserver/src/layered_repository/inmemory_layer.rs +++ b/pageserver/src/layered_repository/inmemory_layer.rs @@ -20,6 +20,7 @@ use std::collections::BTreeMap; use std::ops::Bound::Included; use std::path::PathBuf; use std::sync::{Arc, RwLock}; +use zenith_utils::ordered_vec::OrderedVec; use zenith_utils::accum::Accum; use zenith_utils::lsn::Lsn; @@ -60,7 +61,7 @@ pub struct InMemoryLayerInner { /// /// `segsizes` tracks the size of the segment at different points in time. /// - segsizes: BTreeMap, + segsizes: OrderedVec, /// Writes are only allowed when true. /// Set to false when this layer is in the process of being replaced. @@ -81,9 +82,9 @@ impl InMemoryLayerInner { fn get_seg_size(&self, lsn: Lsn) -> u32 { // Scan the BTreeMap backwards, starting from the given entry. - let mut iter = self.segsizes.range((Included(&Lsn(0)), Included(&lsn))); + let slice = self.segsizes.range((Included(&Lsn(0)), Included(&lsn))); - if let Some((_entry_lsn, entry)) = iter.next_back() { + if let Some((_entry_lsn, entry)) = slice.last() { *entry } else { 0 @@ -343,7 +344,7 @@ impl InMemoryLayer { inner: RwLock::new(InMemoryLayerInner { drop_lsn: None, page_versions: BTreeMap::new(), - segsizes: BTreeMap::new(), + segsizes: OrderedVec::default(), writeable: true, predecessor: None, }), @@ -449,7 +450,7 @@ impl InMemoryLayer { } } - inner.segsizes.insert(lsn, newsize); + inner.segsizes.append_update(lsn, newsize); return Ok(newsize - oldsize); } } @@ -467,12 +468,7 @@ impl InMemoryLayer { let oldsize = inner.get_seg_size(lsn); assert!(segsize < oldsize); - let old = inner.segsizes.insert(lsn, segsize); - - if old.is_some() { - // We already had an entry for this LSN. That's odd.. - warn!("Inserting truncation, but had an entry for the LSN already"); - } + inner.segsizes.append_update(lsn, segsize); Ok(()) } @@ -519,10 +515,10 @@ impl InMemoryLayer { ); // For convenience, copy the segment size from the predecessor layer - let mut segsizes = BTreeMap::new(); + let mut segsizes = OrderedVec::default(); if seg.rel.is_blocky() { let size = src.get_seg_size(start_lsn)?; - segsizes.insert(start_lsn, size); + segsizes.append(start_lsn, size); } Ok(InMemoryLayer { @@ -589,15 +585,16 @@ impl InMemoryLayer { // Divide all the page versions into old and new // at the 'cutoff_lsn' point. - let mut before_segsizes = BTreeMap::new(); - let mut after_segsizes = BTreeMap::new(); + // TODO OrderedVec split method + let mut before_segsizes = OrderedVec::default(); + let mut after_segsizes = OrderedVec::default(); let mut after_oldest_lsn: Accum = Accum(None); for (lsn, size) in inner.segsizes.iter() { if *lsn > cutoff_lsn { - after_segsizes.insert(*lsn, *size); + after_segsizes.append(*lsn, *size); after_oldest_lsn.accum(min, *lsn); } else { - before_segsizes.insert(*lsn, *size); + before_segsizes.append(*lsn, *size); } } @@ -641,7 +638,7 @@ impl InMemoryLayer { let new_inner = new_open.inner.get_mut().unwrap(); new_inner.page_versions.append(&mut after_page_versions); - new_inner.segsizes.append(&mut after_segsizes); + new_inner.segsizes.extend(after_segsizes); Some(Arc::new(new_open)) } else { @@ -705,10 +702,10 @@ impl InMemoryLayer { let end_lsn = self.end_lsn.unwrap(); - let mut before_segsizes = BTreeMap::new(); + let mut before_segsizes = OrderedVec::default(); for (lsn, size) in inner.segsizes.iter() { if *lsn <= end_lsn { - before_segsizes.insert(*lsn, *size); + before_segsizes.append(*lsn, *size); } } let mut before_page_versions = inner.page_versions.iter().filter(|tup| { diff --git a/zenith_utils/src/ordered_vec.rs b/zenith_utils/src/ordered_vec.rs index 14837e0f3d..49b44bedce 100644 --- a/zenith_utils/src/ordered_vec.rs +++ b/zenith_utils/src/ordered_vec.rs @@ -5,7 +5,7 @@ use std::{ use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct OrderedVec(Vec<(K, V)>); impl Default for OrderedVec { @@ -60,6 +60,36 @@ impl OrderedVec { self.0.push((key, value)); } + + pub fn append_update(&mut self, key: K, value: V) { + if let Some((last_key, this_value)) = self.0.last_mut() { + use std::cmp::Ordering; + match (*last_key).cmp(&key) { + Ordering::Less => {} + Ordering::Equal => { + *this_value = value; + return; + } + Ordering::Greater => { + panic!(); + } + } + } + + self.0.push((key, value)); + } + + pub fn extend(&mut self, other: OrderedVec) { + if let (Some((last, _)), Some((first, _))) = (self.0.last(), other.0.first()) { + assert!(last < first); + } + + self.0.extend(other.0); + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } } impl From> for OrderedVec {