compare

Cargo.lock

@@ -615,6 +615,15 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bitmaps"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2"
+dependencies = [
+ "typenum",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.3"
@@ -1762,6 +1771,20 @@ dependencies = [
  "unicode-normalization",
 ]
 
+[[package]]
+name = "im"
+version = "15.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9"
+dependencies = [
+ "bitmaps",
+ "rand_core",
+ "rand_xoshiro",
+ "sized-chunks",
+ "typenum",
+ "version_check",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.9.2"
@@ -2243,6 +2266,7 @@ dependencies = [
  "humantime",
  "humantime-serde",
  "hyper",
+ "im",
  "itertools",
  "metrics",
  "nix",
@@ -2767,6 +2791,15 @@ dependencies = [
  "rand_core",
 ]
 
+[[package]]
+name = "rand_xoshiro"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
+dependencies = [
+ "rand_core",
+]
+
 [[package]]
 name = "rayon"
 version = "1.6.1"
@@ -3454,6 +3487,16 @@ version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
 
+[[package]]
+name = "sized-chunks"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e"
+dependencies = [
+ "bitmaps",
+ "typenum",
+]
+
 [[package]]
 name = "slab"
 version = "0.4.7"

pageserver/Cargo.toml

@@ -69,6 +69,7 @@ utils = { path = "../libs/utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }
 rpds = "0.12.0"
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }
+im = "15.1.0"
 
 [dev-dependencies]
 criterion = "0.4"

pageserver/src/tenant/layer_map.rs

@@ -57,7 +57,7 @@ pub struct LayerMap<L: ?Sized> {
     l0_delta_layers: Vec<Arc<L>>,
 }
 
-impl<L: ?Sized> Default for LayerMap<L> {
+impl<L: ?Sized + PartialEq> Default for LayerMap<L> {
     fn default() -> Self {
         Self {
             open_layer: None,
@@ -77,7 +77,7 @@ pub struct SearchResult<L: ?Sized> {
 
 impl<L> LayerMap<L>
 where
-    L: ?Sized + Layer,
+    L: ?Sized + Layer + PartialEq,
 {
     ///
     /// Find the latest layer (by lsn.end) that covers the given

pageserver/src/tenant/layer_map/historic_layer_coverage.rs

@@ -16,13 +16,13 @@ pub struct HistoricLayerCoverage<Value> {
     historic: BTreeMap<u64, LayerCoverageTuple<Value>>,
 }
 
-impl<T: Clone> Default for HistoricLayerCoverage<T> {
+impl<T: Clone + PartialEq> Default for HistoricLayerCoverage<T> {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl<Value: Clone> HistoricLayerCoverage<Value> {
+impl<Value: Clone + PartialEq> HistoricLayerCoverage<Value> {
     pub fn new() -> Self {
         Self {
             head: LayerCoverageTuple::default(),
@@ -266,13 +266,13 @@ impl<T: std::fmt::Debug> std::fmt::Debug for BufferedHistoricLayerCoverage<T> {
     }
 }
 
-impl<T: Clone> Default for BufferedHistoricLayerCoverage<T> {
+impl<T: Clone + PartialEq> Default for BufferedHistoricLayerCoverage<T> {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl<Value: Clone> BufferedHistoricLayerCoverage<Value> {
+impl<Value: Clone + PartialEq> BufferedHistoricLayerCoverage<Value> {
     pub fn new() -> Self {
         Self {
             historic_coverage: HistoricLayerCoverage::<Value>::new(),

pageserver/src/tenant/layer_map/layer_coverage.rs

@@ -4,6 +4,7 @@ use std::ops::Range;
 // persistent/immutable BTree. It also runs a bit faster but
 // results are not the same on some tests.
 use rpds::RedBlackTreeMapSync;
+use im::OrdMap;
 
 /// Data structure that can efficiently:
 /// - find the latest layer by lsn.end at a given key
@@ -24,18 +25,21 @@ pub struct LayerCoverage<Value> {
     /// be Sync. Using nonsync might be faster, if we can work with
     /// that.
     nodes: RedBlackTreeMapSync<i128, Option<(u64, Value)>>,
+
+    im: OrdMap<i128, Option<(u64, Value)>>,
 }
 
-impl<T: Clone> Default for LayerCoverage<T> {
+impl<T: Clone + PartialEq> Default for LayerCoverage<T> {
     fn default() -> Self {
         Self::new()
     }
 }
 
-impl<Value: Clone> LayerCoverage<Value> {
+impl<Value: Clone + PartialEq> LayerCoverage<Value> {
     pub fn new() -> Self {
         Self {
             nodes: RedBlackTreeMapSync::default(),
+            im: OrdMap::default(),
         }
     }
 
@@ -49,6 +53,13 @@ impl<Value: Clone> LayerCoverage<Value> {
             None => None,
         };
         self.nodes.insert_mut(key, value);
+
+        let im_value = match self.im.range(..=key).last() {
+            Some((_, Some(v))) => Some(v.clone()),
+            Some((_, None)) => None,
+            None => None,
+        };
+        self.im.insert(key, im_value);
     }
 
     /// Insert a layer.
@@ -93,19 +104,56 @@ impl<Value: Clone> LayerCoverage<Value> {
         for k in to_remove {
             self.nodes.remove_mut(&k);
         }
+
+        let mut to_update = Vec::new();
+        let mut to_remove = Vec::new();
+        let mut prev_covered = false;
+        for (k, node) in self.im.range(key.clone()) {
+            let needs_cover = match node {
+                None => true,
+                Some((h, _)) => h < &lsn.end,
+            };
+            if needs_cover {
+                match prev_covered {
+                    true => to_remove.push(*k),
+                    false => to_update.push(*k),
+                }
+            }
+            prev_covered = needs_cover;
+        }
+        if !prev_covered {
+            to_remove.push(key.end);
+        }
+        for k in to_update {
+            self.im.insert(k, Some((lsn.end, value.clone())));
+        }
+        for k in to_remove {
+            self.im.remove(&k);
+        }
     }
 
     /// Get the latest (by lsn.end) layer at a given key
     ///
     /// Complexity: O(log N)
     pub fn query(&self, key: i128) -> Option<Value> {
-        self.nodes
+        let res = self.nodes
             .range(..=key)
             .rev()
             .next()?
             .1
             .as_ref()
-            .map(|(_, v)| v.clone())
+            .map(|(_, v)| v.clone());
+        let im_res = self.nodes
+            .range(..=key)
+            .rev()
+            .next()?
+            .1
+            .as_ref()
+            .map(|(_, v)| v.clone());
+        if res != im_res {
+            panic!("aha");
+        }
+        res
     }
 
     /// Iterate the changes in layer coverage in a given range. You will likely
@@ -122,6 +170,7 @@ impl<Value: Clone> LayerCoverage<Value> {
     pub fn clone(&self) -> Self {
         Self {
             nodes: self.nodes.clone(),
+            im: self.im.clone(),
         }
     }
 }
@@ -132,7 +181,7 @@ pub struct LayerCoverageTuple<Value> {
     pub delta_coverage: LayerCoverage<Value>,
 }
 
-impl<T: Clone> Default for LayerCoverageTuple<T> {
+impl<T: Clone + PartialEq> Default for LayerCoverageTuple<T> {
     fn default() -> Self {
         Self {
             image_coverage: LayerCoverage::default(),
@@ -141,7 +190,7 @@ impl<T: Clone> Default for LayerCoverageTuple<T> {
     }
 }
 
-impl<Value: Clone> LayerCoverageTuple<Value> {
+impl<Value: Clone + PartialEq> LayerCoverageTuple<Value> {
     pub fn clone(&self) -> Self {
         Self {
             image_coverage: self.image_coverage.clone(),

pageserver/src/tenant/storage_layer.rs

@@ -187,6 +187,12 @@ pub trait PersistentLayer: Layer {
     fn file_size(&self) -> Option<u64>;
 }
 
+impl PartialEq for dyn PersistentLayer {
+    fn eq(&self, other: &Self) -> bool {
+        self.filename().eq(&other.filename())
+    }
+}
+
 pub fn downcast_remote_layer(
     layer: &Arc<dyn PersistentLayer>,
 ) -> Option<std::sync::Arc<RemoteLayer>> {
@@ -213,6 +219,12 @@ pub struct LayerDescriptor {
     pub short_id: String,
 }
 
+impl PartialEq for LayerDescriptor {
+    fn eq(&self, other: &Self) -> bool {
+        self.short_id == other.short_id
+    }
+}
+
 impl Layer for LayerDescriptor {
     fn get_key_range(&self) -> Range<Key> {
         self.key.clone()

test_runner/performance/test_perf_pgbench.py

@@ -172,8 +172,10 @@ def get_scales_matrix(default: int = 10) -> List[int]:
 @pytest.mark.parametrize("duration", get_durations_matrix())
 def test_pgbench(neon_with_baseline: PgCompare, scale: int, duration: int):
     run_test_pgbench(neon_with_baseline, scale, duration, PgBenchLoadType.INIT)
-    run_test_pgbench(neon_with_baseline, scale, duration, PgBenchLoadType.SIMPLE_UPDATE)
-    run_test_pgbench(neon_with_baseline, scale, duration, PgBenchLoadType.SELECT_ONLY)
+    run_test_pgbench(neon_with_baseline, scale, duration, PgBenchLoadType.INIT)
+    run_test_pgbench(neon_with_baseline, scale, duration, PgBenchLoadType.INIT)
+    # run_test_pgbench(neon_with_baseline, scale, duration, PgBenchLoadType.SIMPLE_UPDATE)
+    # run_test_pgbench(neon_with_baseline, scale, duration, PgBenchLoadType.SELECT_ONLY)
 
 
 # The following 3 tests run on an existing database as it was set up by previous tests,