From 72835371cc832cd61f75e897201eeaf8c37466c1 Mon Sep 17 00:00:00 2001 From: Bojan Serafimov Date: Thu, 1 Dec 2022 20:49:43 -0500 Subject: [PATCH] Add mock persistent BST implementation --- Cargo.lock | 25 ++++++++++++++ pageserver/Cargo.toml | 1 + pageserver/benches/bench_layer_map.rs | 24 +++++++++++++ pageserver/src/tenant.rs | 1 + pageserver/src/tenant/bst_layer_map.rs | 47 ++++++++++++++++++++++++++ 5 files changed, 98 insertions(+) create mode 100644 pageserver/src/tenant/bst_layer_map.rs diff --git a/Cargo.lock b/Cargo.lock index 8194e107ef..6e59d46339 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,6 +66,15 @@ dependencies = [ "backtrace", ] +[[package]] +name = "archery" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a8da9bc4c4053ee067669762bcaeea6e241841295a2b6c948312dad6ef4cc02" +dependencies = [ + "static_assertions", +] + [[package]] name = "arrayvec" version = "0.7.2" @@ -2346,6 +2355,7 @@ dependencies = [ "rand", "regex", "remote_storage", + "rpds", "rstar", "scopeguard", "serde", @@ -3163,6 +3173,15 @@ dependencies = [ "regex", ] +[[package]] +name = "rpds" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66262ea963eff99163e6b741fbc3417a52cc13074728c1047e9911789df9b000" +dependencies = [ + "archery", +] + [[package]] name = "rstar" version = "0.9.3" @@ -3633,6 +3652,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "storage_broker" version = "0.1.0" diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 6afebf6842..2edf503ee1 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -70,6 +70,7 @@ remote_storage = { path = "../libs/remote_storage" } tenant_size_model = { path = "../libs/tenant_size_model" } utils = { path = "../libs/utils" } workspace_hack = { version = "0.1", path = "../workspace_hack" } +rpds = "0.12.0" [dev-dependencies] criterion = "0.4" diff --git a/pageserver/benches/bench_layer_map.rs b/pageserver/benches/bench_layer_map.rs index 33a0fe1fe6..21273f8dd9 100644 --- a/pageserver/benches/bench_layer_map.rs +++ b/pageserver/benches/bench_layer_map.rs @@ -1,5 +1,6 @@ use anyhow::Result; use pageserver::repository::{Key, Value}; +use pageserver::tenant::bst_layer_map::BSTLM; use pageserver::tenant::filename::{DeltaFileName, ImageFileName}; use pageserver::tenant::layer_map::LayerMap; use pageserver::tenant::segment_tree_layer_map::STLM; @@ -261,6 +262,7 @@ fn bench_from_real_project(c: &mut Criterion) { fn bench_sequential(c: &mut Criterion) { let mut layer_map = LayerMap::default(); let mut stlm = STLM::new(); + let mut bstlm = BSTLM::new(); // Init layer map. Create 100_000 layers arranged in 1000 diagonal lines. // @@ -298,6 +300,21 @@ fn bench_sequential(c: &mut Criterion) { } println!("Finished persistent segment tree init in {:?}", now.elapsed()); + let now = Instant::now(); + for i in 0..100_000 { + // TODO try inserting a super-wide layer in between every 10 to reflect + // what often happens with L1 layers that include non-rel changes. + // Maybe do that as a separate test. + let i32 = (i as u32) % 100; + let zero = Key::from_hex("000000000000000000000000000000000000").unwrap(); + let layer = DummyImage { + key_range: zero.add(10 * i32)..zero.add(10 * i32 + 1), + lsn: Lsn(i), + }; + bstlm.insert(10 * i32, 10 * i32 + 1, i as u32, format!("Layer {}", i)); + } + println!("Finished bst init in {:?}", now.elapsed()); + // Choose 100 uniformly random queries let rng = &mut StdRng::seed_from_u64(1); let queries: Vec<(Key, Lsn)> = uniform_query_pattern(&layer_map) @@ -321,6 +338,13 @@ fn bench_sequential(c: &mut Criterion) { } }); }); + group.bench_function("persistent_bst", |b| { + b.iter(|| { + for q in queries.clone().into_iter() { + bstlm.query(q.0.field6, q.1.0 as u32); + } + }); + }); group.finish(); } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index e2c014053f..8e74e099dc 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -80,6 +80,7 @@ pub mod filename; mod image_layer; mod inmemory_layer; pub mod layer_map; +pub mod bst_layer_map; pub mod segment_tree_layer_map; pub mod metadata; diff --git a/pageserver/src/tenant/bst_layer_map.rs b/pageserver/src/tenant/bst_layer_map.rs new file mode 100644 index 0000000000..e06000f8ea --- /dev/null +++ b/pageserver/src/tenant/bst_layer_map.rs @@ -0,0 +1,47 @@ +use std::collections::BTreeMap; + +use rpds::RedBlackTreeMap; + + +pub struct BSTLM { + head: RedBlackTreeMap, + historic: BTreeMap>, +} + +/// Layer map (good enough for benchmarks) implemented using persistent segment tree +impl BSTLM { + pub fn new() -> Self { + BSTLM { + head: RedBlackTreeMap::default(), + historic: BTreeMap::default(), + } + } + + pub fn insert(self: &mut Self, key_begin: u32, key_end: u32, lsn: u32, value: String) { + self.head.insert_mut(key_begin, value.clone()); + self.head.insert_mut(key_end, value.clone()); // TODO wrong value + // TODO also remove what's in between + self.historic.insert(lsn, self.head.clone()); // O(1) clone + } + + pub fn query(self: &Self, key: u32, lsn: u32) -> Option<&String> { + let version = self.historic.range(0..=lsn).rev().next()?.1; + Some(version.range(key..).next()?.1) + } +} + + +fn test_bstlm() { + let mut bstlm = BSTLM::new(); + bstlm.insert(0, 5, 100, "layer 1".to_string()); + bstlm.insert(3, 9, 110, "layer 2".to_string()); + + dbg!(bstlm.query(1, 105)); + dbg!(bstlm.query(4, 105)); + dbg!(bstlm.query(4, 115)); +} + +#[test] +fn test_bstlm_() { + test_bstlm() +}