rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-08 05:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

pagebench: fixup after is_rel_block_key changes in #6266 (#6303)

Browse Source 
PR #6266 broke the getpage_latest_lsn benchmark.

Before this patch, we'd fail with

```
not implemented: split up range
```

because `r.start = rel size key` and `r.end = rel size key + 1`.

The filtering of the key ranges in that loop is a bit ugly, but,
I measured:
* setup with 180k layer files (20k tenants * 9 layers).
* total physical size is 463GiB
* 5k tenants, the range filtering takes `0.6 seconds` on an
i3en.3xlarge.
That's a tiny fraction of the overall time it takes for pagebench to get
ready to send requests. So, this is good enough for now / there are
other bottlenecks that are bigger.
This commit is contained in:
Christian Schwarz
2024-01-09 19:00:37 +01:00
committed by GitHub
parent f94abbab95
commit 4e1b0b84eb
3 changed files with 33 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
pageserver/pagebench/src/cmd/getpage_latest_lsn.rs
View File

@@ -3,6 +3,7 @@ use futures::future::join_all;
use pageserver::pgdatadir_mapping::key_to_rel_block;
use pageserver::repository;
use pageserver_api::key::is_rel_block_key;
use pageserver_api::keyspace::KeySpaceAccum;
use pageserver_api::models::PagestreamGetPageRequest;
use utils::id::TenantTimelineId;
@@ -116,38 +117,42 @@ async fn main_impl(
.keyspace(timeline.tenant_id, timeline.timeline_id)
.await?;
let lsn = partitioning.at_lsn;
let ranges = partitioning
.keys
.ranges
.iter()
.filter_map(|r| {
let start = r.start;
let end = r.end;
// filter out non-relblock keys
match (is_rel_block_key(&start), is_rel_block_key(&end)) {
(true, true) => Some(KeyRange {
timeline,
timeline_lsn: lsn,
start: start.to_i128(),
end: end.to_i128(),
}),
(true, false) | (false, true) => {
unimplemented!("split up range")
}
(false, false) => None,
let start = Instant::now();
let mut filtered = KeySpaceAccum::new();
// let's hope this is inlined and vectorized...
// TODO: turn this loop into a is_rel_block_range() function.
for r in partitioning.keys.ranges.iter() {
let mut i = r.start;
while i != r.end {
if is_rel_block_key(&i) {
filtered.add_key(i);
}
})
.collect::<Vec<_>>();
i = i.next();
}
}
let filtered = filtered.to_keyspace();
let filter_duration = start.elapsed();
anyhow::Ok(ranges)
anyhow::Ok((
filter_duration,
filtered.ranges.into_iter().map(move |r| KeyRange {
timeline,
timeline_lsn: lsn,
start: r.start.to_i128(),
end: r.end.to_i128(),
}),
))
}
});
}
let mut total_filter_duration = Duration::from_secs(0);
let mut all_ranges: Vec<KeyRange> = Vec::new();
while let Some(res) = js.join_next().await {
all_ranges.extend(res.unwrap().unwrap());
let (filter_duration, range) = res.unwrap().unwrap();
all_ranges.extend(range);
total_filter_duration += filter_duration;
}
info!("filter duration: {}", total_filter_duration.as_secs_f64());
let live_stats = Arc::new(LiveStats::default());
@@ -256,6 +261,7 @@ async fn main_impl(
let r = &ranges[weights.sample(&mut rng)];
let key: i128 = rng.gen_range(r.start..r.end);
let key = repository::Key::from_i128(key);
assert!(is_rel_block_key(&key));
let (rel_tag, block_no) = key_to_rel_block(key)
.expect("we filter non-rel-block keys out above");
PagestreamGetPageRequest {