From cb5b9375d2fc14f08f35d7d23cc3e031ec086a92 Mon Sep 17 00:00:00 2001 From: Bojan Serafimov Date: Tue, 10 Jan 2023 21:03:26 -0500 Subject: [PATCH] Organize modules, rename structs --- pageserver/src/tenant.rs | 3 - pageserver/src/tenant/latest_layer_map.rs | 28 --------- pageserver/src/tenant/layer_map.rs | 25 ++++---- .../historic_layer_coverage.rs} | 61 ++++++++++--------- .../tenant/{ => layer_map}/layer_coverage.rs | 28 +++++++++ 5 files changed, 73 insertions(+), 72 deletions(-) delete mode 100644 pageserver/src/tenant/latest_layer_map.rs rename pageserver/src/tenant/{bst_layer_map.rs => layer_map/historic_layer_coverage.rs} (89%) rename pageserver/src/tenant/{ => layer_map}/layer_coverage.rs (83%) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index ee8a1ad2e7..d74f263f08 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -74,11 +74,8 @@ use utils::{ mod blob_io; pub mod block_io; -pub mod bst_layer_map; mod disk_btree; pub(crate) mod ephemeral_file; -pub mod latest_layer_map; -pub mod layer_coverage; pub mod layer_map; pub mod metadata; diff --git a/pageserver/src/tenant/latest_layer_map.rs b/pageserver/src/tenant/latest_layer_map.rs deleted file mode 100644 index 8b625862e4..0000000000 --- a/pageserver/src/tenant/latest_layer_map.rs +++ /dev/null @@ -1,28 +0,0 @@ -use super::layer_coverage::LayerCoverage; - -/// Separate coverage data structure for each layer type. -/// -/// This is just a tuple but with the elements named to -/// prevent bugs. -pub struct LatestLayerMap { - pub image_coverage: LayerCoverage, - pub delta_coverage: LayerCoverage, -} - -impl Default for LatestLayerMap { - fn default() -> Self { - Self { - image_coverage: LayerCoverage::default(), - delta_coverage: LayerCoverage::default(), - } - } -} - -impl LatestLayerMap { - pub fn clone(self: &Self) -> Self { - Self { - image_coverage: self.image_coverage.clone(), - delta_coverage: self.delta_coverage.clone(), - } - } -} diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index 34c80951cf..c96e804135 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -10,6 +10,9 @@ //! corresponding files are written to disk. //! +mod historic_layer_coverage; +mod layer_coverage; + use crate::keyspace::KeyPartitioning; use crate::metrics::NUM_ONDISK_LAYERS; use crate::repository::Key; @@ -21,7 +24,7 @@ use std::ops::Range; use std::sync::Arc; use utils::lsn::Lsn; -use super::bst_layer_map::RetroactiveLayerMap; +use historic_layer_coverage::BufferedHistoricLayerCoverage; /// /// LayerMap tracks what layers exist on a timeline. @@ -47,7 +50,7 @@ pub struct LayerMap { pub frozen_layers: VecDeque>, /// Index of the historic layers optimized for search - index: RetroactiveLayerMap>, + historic: BufferedHistoricLayerCoverage>, /// L0 layers have key range Key::MIN..Key::MAX, and locating them using R-Tree search is very inefficient. /// So L0 layers are held in l0_delta_layers vector, in addition to the R-tree. @@ -61,7 +64,7 @@ impl Default for LayerMap { next_open_layer_at: None, frozen_layers: VecDeque::default(), l0_delta_layers: Vec::default(), - index: RetroactiveLayerMap::default(), + historic: BufferedHistoricLayerCoverage::default(), } } } @@ -91,7 +94,7 @@ where /// 'open' and 'frozen' layers! /// pub fn search(&self, key: Key, end_lsn: Lsn) -> Option> { - let version = self.index.get().unwrap().get_version(end_lsn.0 - 1)?; + let version = self.historic.get().unwrap().get_version(end_lsn.0 - 1)?; let latest_delta = version.delta_coverage.query(key.to_i128()); let latest_image = version.image_coverage.query(key.to_i128()); @@ -138,7 +141,7 @@ where pub fn insert_historic(&mut self, layer: Arc) { let kr = layer.get_key_range(); let lr = layer.get_lsn_range(); - self.index.insert( + self.historic.insert( kr.start.to_i128()..kr.end.to_i128(), lr.start.0..lr.end.0, Arc::clone(&layer), @@ -154,7 +157,7 @@ where /// Must be called after a batch of insert_historic calls, before querying pub fn rebuild_index(&mut self) { - self.index.rebuild(); + self.historic.rebuild(); } /// @@ -165,7 +168,7 @@ where pub fn remove_historic(&mut self, layer: Arc) { let kr = layer.get_key_range(); let lr = layer.get_lsn_range(); - self.index.remove( + self.historic.remove( kr.start.to_i128()..kr.end.to_i128(), lr.start.0..lr.end.0, !layer.is_incremental(), @@ -196,7 +199,7 @@ where return Ok(true); } - let version = match self.index.get().unwrap().get_version(lsn.end.0) { + let version = match self.historic.get().unwrap().get_version(lsn.end.0) { Some(v) => v, None => return Ok(false), }; @@ -225,7 +228,7 @@ where } pub fn iter_historic_layers(&self) -> impl '_ + Iterator> { - self.index.iter() + self.historic.iter() } /// @@ -241,7 +244,7 @@ where key_range: &Range, lsn: Lsn, ) -> Result, Option>)>> { - let version = match self.index.get().unwrap().get_version(lsn.0 - 1) { + let version = match self.historic.get().unwrap().get_version(lsn.0 - 1) { Some(v) => v, None => return Ok(vec![]), }; @@ -283,7 +286,7 @@ where return Ok(0); } - let version = match self.index.get().unwrap().get_version(lsn.end.0 - 1) { + let version = match self.historic.get().unwrap().get_version(lsn.end.0 - 1) { Some(v) => v, None => return Ok(0), }; diff --git a/pageserver/src/tenant/bst_layer_map.rs b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs similarity index 89% rename from pageserver/src/tenant/bst_layer_map.rs rename to pageserver/src/tenant/layer_map/historic_layer_coverage.rs index 48fbe451a3..aaa59700e3 100644 --- a/pageserver/src/tenant/bst_layer_map.rs +++ b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs @@ -1,26 +1,31 @@ use std::collections::BTreeMap; use std::ops::Range; -use super::latest_layer_map::LatestLayerMap; +use super::layer_coverage::LayerCoverageTuple; -pub struct PersistentLayerMap { - /// The latest-only solution - head: LatestLayerMap, +/// Efficiently queryable layer coverage for each LSN. +/// +/// Allows answering layer map queries very efficiently, +/// but doesn't allow retroactive insertion, which is +/// sometimes necessary. See BufferedHistoricLayerCoverage. +pub struct HistoricLayerCoverage { + /// The latest state + head: LayerCoverageTuple, /// All previous states - historic: BTreeMap>, + historic: BTreeMap>, } -impl Default for PersistentLayerMap { +impl Default for HistoricLayerCoverage { fn default() -> Self { Self::new() } } -impl PersistentLayerMap { +impl HistoricLayerCoverage { pub fn new() -> Self { Self { - head: LatestLayerMap::default(), + head: LayerCoverageTuple::default(), historic: BTreeMap::default(), } } @@ -56,7 +61,7 @@ impl PersistentLayerMap { self.historic.insert(lsn.start, self.head.clone()); } - pub fn get_version(self: &Self, lsn: u64) -> Option<&LatestLayerMap> { + pub fn get_version(self: &Self, lsn: u64) -> Option<&LayerCoverageTuple> { match self.historic.range(..=lsn).rev().next() { Some((_, v)) => Some(v), None => None, @@ -79,7 +84,7 @@ impl PersistentLayerMap { /// All layers in this test have height 1. #[test] fn test_persistent_simple() { - let mut map = PersistentLayerMap::::new(); + let mut map = HistoricLayerCoverage::::new(); map.insert(0..5, 100..101, "Layer 1".to_string(), true); map.insert(3..9, 110..111, "Layer 2".to_string(), true); map.insert(5..6, 120..121, "Layer 3".to_string(), true); @@ -105,7 +110,7 @@ fn test_persistent_simple() { /// Cover simple off-by-one edge cases #[test] fn test_off_by_one() { - let mut map = PersistentLayerMap::::new(); + let mut map = HistoricLayerCoverage::::new(); map.insert(3..5, 100..110, "Layer 1".to_string(), true); // Check different LSNs @@ -127,7 +132,7 @@ fn test_off_by_one() { /// Cover edge cases where layers begin or end on the same key #[test] fn test_key_collision() { - let mut map = PersistentLayerMap::::new(); + let mut map = HistoricLayerCoverage::::new(); map.insert(3..5, 100..110, "Layer 10".to_string(), true); map.insert(5..8, 100..110, "Layer 11".to_string(), true); @@ -172,7 +177,7 @@ fn test_key_collision() { /// Test when rectangles have nontrivial height and possibly overlap #[test] fn test_persistent_overlapping() { - let mut map = PersistentLayerMap::::new(); + let mut map = HistoricLayerCoverage::::new(); // Add 3 key-disjoint layers with varying LSN ranges map.insert(1..2, 100..200, "Layer 1".to_string(), true); @@ -219,7 +224,7 @@ fn test_persistent_overlapping() { assert_eq!(version.image_coverage.query(8), Some("Layer 6".to_string())); } -/// Wrapper for PersistentLayerMap that allows us to hack around the lack +/// Wrapper for HistoricLayerCoverage that allows us to hack around the lack /// of support for retroactive insertion by rebuilding the map since the /// change. /// @@ -238,9 +243,9 @@ fn test_persistent_overlapping() { /// /// See this for more on persistent and retroactive techniques: /// https://www.youtube.com/watch?v=WqCWghETNDc&t=581s -pub struct RetroactiveLayerMap { +pub struct BufferedHistoricLayerCoverage { /// A persistent layer map that we rebuild when we need to retroactively update - map: PersistentLayerMap, + historic_coverage: HistoricLayerCoverage, /// We buffer insertion into the PersistentLayerMap to decrease the number of rebuilds. /// @@ -254,7 +259,7 @@ pub struct RetroactiveLayerMap { layers: BTreeMap<(u64, u64, i128, i128, bool), Value>, } -impl std::fmt::Debug for RetroactiveLayerMap { +impl std::fmt::Debug for BufferedHistoricLayerCoverage { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RetroactiveLayerMap") .field("buffer", &self.buffer) @@ -263,16 +268,16 @@ impl std::fmt::Debug for RetroactiveLayerMap { } } -impl Default for RetroactiveLayerMap { +impl Default for BufferedHistoricLayerCoverage { fn default() -> Self { Self::new() } } -impl RetroactiveLayerMap { +impl BufferedHistoricLayerCoverage { pub fn new() -> Self { Self { - map: PersistentLayerMap::::new(), + historic_coverage: HistoricLayerCoverage::::new(), buffer: BTreeMap::new(), layers: BTreeMap::new(), } @@ -324,11 +329,11 @@ impl RetroactiveLayerMap { }); // Rebuild - self.map.trim(&rebuild_since); + self.historic_coverage.trim(&rebuild_since); for ((lsn_start, lsn_end, key_start, key_end, is_image), layer) in self.layers.range((rebuild_since, 0, 0, 0, false)..) { - self.map.insert( + self.historic_coverage.insert( *key_start..*key_end, *lsn_start..*lsn_end, layer.clone(), @@ -337,10 +342,6 @@ impl RetroactiveLayerMap { } } - pub fn clear(self: &mut Self) { - self.map.trim(&0); - } - /// Iterate all the layers pub fn iter(self: &Self) -> impl '_ + Iterator { // NOTE we can actually perform this without rebuilding, @@ -354,7 +355,7 @@ impl RetroactiveLayerMap { /// Return a reference to a queryable map, assuming all updates /// have already been processed using self.rebuild() - pub fn get(self: &Self) -> anyhow::Result<&PersistentLayerMap> { + pub fn get(self: &Self) -> anyhow::Result<&HistoricLayerCoverage> { // NOTE we error here instead of implicitly rebuilding because // rebuilding is somewhat expensive. // TODO maybe implicitly rebuild and log/sentry an error? @@ -362,13 +363,13 @@ impl RetroactiveLayerMap { anyhow::bail!("rebuild required") } - Ok(&self.map) + Ok(&self.historic_coverage) } } #[test] fn test_retroactive_regression_1() { - let mut map = RetroactiveLayerMap::new(); + let mut map = BufferedHistoricLayerCoverage::new(); map.insert( 0..21267647932558653966460912964485513215, @@ -388,7 +389,7 @@ fn test_retroactive_regression_1() { #[test] fn test_retroactive_simple() { - let mut map = RetroactiveLayerMap::new(); + let mut map = BufferedHistoricLayerCoverage::new(); // Append some images in increasing LSN order map.insert(0..5, 100..101, "Image 1".to_string(), true); diff --git a/pageserver/src/tenant/layer_coverage.rs b/pageserver/src/tenant/layer_map/layer_coverage.rs similarity index 83% rename from pageserver/src/tenant/layer_coverage.rs rename to pageserver/src/tenant/layer_map/layer_coverage.rs index 14394d2b29..7bbdc2fd44 100644 --- a/pageserver/src/tenant/layer_coverage.rs +++ b/pageserver/src/tenant/layer_map/layer_coverage.rs @@ -16,6 +16,10 @@ pub struct LayerCoverage { /// For every change in coverage (as we sweep the key space) /// we store (lsn.end, value). /// + /// We use an immutable/persistent tree so that we can keep historic + /// versions of this coverage without cloning the whole thing and + /// incurring quadratic memory cost. See HistoricLayerCoverage. + /// /// We use the Sync version of the map because we want Self to /// be Sync. Using nonsync might be faster, if we can work with /// that. @@ -132,3 +136,27 @@ impl LayerCoverage { } } } + +/// Image and delta coverage at a specific LSN. +pub struct LayerCoverageTuple { + pub image_coverage: LayerCoverage, + pub delta_coverage: LayerCoverage, +} + +impl Default for LayerCoverageTuple { + fn default() -> Self { + Self { + image_coverage: LayerCoverage::default(), + delta_coverage: LayerCoverage::default(), + } + } +} + +impl LayerCoverageTuple { + pub fn clone(self: &Self) -> Self { + Self { + image_coverage: self.image_coverage.clone(), + delta_coverage: self.delta_coverage.clone(), + } + } +}