Add mock persistent BST implementation

Cargo.lock

@@ -66,6 +66,15 @@ dependencies = [
  "backtrace",
 ]
 
+[[package]]
+name = "archery"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a8da9bc4c4053ee067669762bcaeea6e241841295a2b6c948312dad6ef4cc02"
+dependencies = [
+ "static_assertions",
+]
+
 [[package]]
 name = "arrayvec"
 version = "0.7.2"
@@ -2346,6 +2355,7 @@ dependencies = [
  "rand",
  "regex",
  "remote_storage",
+ "rpds",
  "rstar",
  "scopeguard",
  "serde",
@@ -3163,6 +3173,15 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "rpds"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66262ea963eff99163e6b741fbc3417a52cc13074728c1047e9911789df9b000"
+dependencies = [
+ "archery",
+]
+
 [[package]]
 name = "rstar"
 version = "0.9.3"
@@ -3633,6 +3652,12 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
+[[package]]
+name = "static_assertions"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
+
 [[package]]
 name = "storage_broker"
 version = "0.1.0"

pageserver/Cargo.toml

@@ -70,6 +70,7 @@ remote_storage = { path = "../libs/remote_storage" }
 tenant_size_model = { path = "../libs/tenant_size_model" }
 utils = { path = "../libs/utils" }
 workspace_hack = { version = "0.1", path = "../workspace_hack" }
+rpds = "0.12.0"
 
 [dev-dependencies]
 criterion = "0.4"

pageserver/benches/bench_layer_map.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
 use pageserver::repository::{Key, Value};
+use pageserver::tenant::bst_layer_map::BSTLM;
 use pageserver::tenant::filename::{DeltaFileName, ImageFileName};
 use pageserver::tenant::layer_map::LayerMap;
 use pageserver::tenant::segment_tree_layer_map::STLM;
@@ -261,6 +262,7 @@ fn bench_from_real_project(c: &mut Criterion) {
 fn bench_sequential(c: &mut Criterion) {
     let mut layer_map = LayerMap::default();
     let mut stlm = STLM::new();
+    let mut bstlm = BSTLM::new();
 
     // Init layer map. Create 100_000 layers arranged in 1000 diagonal lines.
     //
@@ -298,6 +300,21 @@ fn bench_sequential(c: &mut Criterion) {
     }
     println!("Finished persistent segment tree init in {:?}", now.elapsed());
 
+    let now = Instant::now();
+    for i in 0..100_000 {
+        // TODO try inserting a super-wide layer in between every 10 to reflect
+        //      what often happens with L1 layers that include non-rel changes.
+        //      Maybe do that as a separate test.
+        let i32 = (i as u32) % 100;
+        let zero = Key::from_hex("000000000000000000000000000000000000").unwrap();
+        let layer = DummyImage {
+            key_range: zero.add(10 * i32)..zero.add(10 * i32 + 1),
+            lsn: Lsn(i),
+        };
+        bstlm.insert(10 * i32, 10 * i32 + 1, i as u32, format!("Layer {}", i));
+    }
+    println!("Finished bst init in {:?}", now.elapsed());
+
     // Choose 100 uniformly random queries
     let rng = &mut StdRng::seed_from_u64(1);
     let queries: Vec<(Key, Lsn)> = uniform_query_pattern(&layer_map)
@@ -321,6 +338,13 @@ fn bench_sequential(c: &mut Criterion) {
             }
         });
     });
+    group.bench_function("persistent_bst", |b| {
+        b.iter(|| {
+            for q in queries.clone().into_iter() {
+                bstlm.query(q.0.field6, q.1.0 as u32);
+            }
+        });
+    });
     group.finish();
 }
 

pageserver/src/tenant.rs

@@ -80,6 +80,7 @@ pub mod filename;
 mod image_layer;
 mod inmemory_layer;
 pub mod layer_map;
+pub mod bst_layer_map;
 pub mod segment_tree_layer_map;
 
 pub mod metadata;

pageserver/src/tenant/bst_layer_map.rs

@@ -0,0 +1,47 @@
+use std::collections::BTreeMap;
+
+use rpds::RedBlackTreeMap;
+
+
+pub struct BSTLM {
+    head: RedBlackTreeMap<u32, String>,
+    historic: BTreeMap<u32, RedBlackTreeMap<u32, String>>,
+}
+
+/// Layer map (good enough for benchmarks) implemented using persistent segment tree
+impl BSTLM {
+    pub fn new() -> Self {
+        BSTLM {
+            head: RedBlackTreeMap::default(),
+            historic: BTreeMap::default(),
+        }
+    }
+
+    pub fn insert(self: &mut Self, key_begin: u32, key_end: u32, lsn: u32, value: String) {
+        self.head.insert_mut(key_begin, value.clone());
+        self.head.insert_mut(key_end, value.clone()); // TODO wrong value
+        // TODO also remove what's in between
+        self.historic.insert(lsn, self.head.clone());  // O(1) clone
+    }
+
+    pub fn query(self: &Self, key: u32, lsn: u32) -> Option<&String> {
+        let version = self.historic.range(0..=lsn).rev().next()?.1;
+        Some(version.range(key..).next()?.1)
+    }
+}
+
+
+fn test_bstlm() {
+    let mut bstlm = BSTLM::new();
+    bstlm.insert(0, 5, 100, "layer 1".to_string());
+    bstlm.insert(3, 9, 110, "layer 2".to_string());
+
+    dbg!(bstlm.query(1, 105));
+    dbg!(bstlm.query(4, 105));
+    dbg!(bstlm.query(4, 115));
+}
+
+#[test]
+fn test_bstlm_() {
+    test_bstlm()
+}