rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-10 06:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

pageserver - track size of VecMaps

Browse Source
This commit is contained in:
Patrick Insinger
2021-11-02 12:47:59 -07:00
committed by Patrick Insinger
parent 9300107cdf
commit 1ce4976e36
3 changed files with 53 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
pageserver/src/layered_repository/inmemory_layer.rs
View File

@@ -386,7 +386,8 @@ impl InMemoryLayer {
PageVersion::Wal(rec) => rec.rec.len(),
};
let old = inner.page_versions.append_or_update_last(blknum, lsn, pv);
let (old, delta_size) = inner.page_versions.append_or_update_last(blknum, lsn, pv);
mem_usage += delta_size;
if old.is_some() {
// We already had an entry for this LSN. That's odd..
@@ -396,8 +397,7 @@ impl InMemoryLayer {
);
}
inner.mem_usage += mem_usage;
GLOBAL_OPEN_MEM_USAGE.fetch_add(mem_usage, Ordering::Relaxed);
let mut delta_logical_size = 0;
// Also update the relation size, if this extended the relation.
if self.seg.rel.is_blocky() {
@@ -431,9 +431,10 @@ impl InMemoryLayer {
gapblknum,
blknum
);
let old = inner
let (old, delta_size) = inner
.page_versions
.append_or_update_last(gapblknum, lsn, zeropv);
mem_usage += delta_size;
// We already had an entry for this LSN. That's odd..
if old.is_some() {
@@ -444,12 +445,18 @@ impl InMemoryLayer {
}
}
inner.segsizes.append_or_update_last(lsn, newsize).unwrap();
return newsize - oldsize;
let (_old, delta_size) =
inner.segsizes.append_or_update_last(lsn, newsize).unwrap();
mem_usage += delta_size;
delta_logical_size = newsize - oldsize;
}
}
0
inner.mem_usage += mem_usage;
GLOBAL_OPEN_MEM_USAGE.fetch_add(mem_usage, Ordering::Relaxed);
delta_logical_size
}
/// Remember that the relation was truncated at given LSN
@@ -466,7 +473,9 @@ impl InMemoryLayer {
let oldsize = inner.get_seg_size(lsn);
assert!(segsize < oldsize);
let old = inner.segsizes.append_or_update_last(lsn, segsize).unwrap();
let (old, delta_size) = inner.segsizes.append_or_update_last(lsn, segsize).unwrap();
inner.mem_usage += delta_size;
GLOBAL_OPEN_MEM_USAGE.fetch_add(delta_size, Ordering::Relaxed);
if old.is_some() {
// We already had an entry for this LSN. That's odd..

4
pageserver/src/layered_repository/page_versions.rs
View File

@@ -15,7 +15,7 @@ impl PageVersions {
blknum: u32,
lsn: Lsn,
page_version: PageVersion,
) -> Option<PageVersion> {
) -> (Option<PageVersion>, usize) {
let map = self.0.entry(blknum).or_insert_with(VecMap::default);
map.append_or_update_last(lsn, page_version).unwrap()
}
@@ -119,7 +119,7 @@ mod tests {
for blknum in 0..BLOCKS {
for lsn in 0..LSNS {
let old = page_versions.append_or_update_last(
let (old, _delta_size) = page_versions.append_or_update_last(
blknum,
Lsn(lsn),
empty_page_version.clone(),

45
zenith_utils/src/vec_map.rs
View File

@@ -1,4 +1,4 @@
use std::{cmp::Ordering, ops::RangeBounds};
use std::{alloc::Layout, cmp::Ordering, ops::RangeBounds};
use serde::{Deserialize, Serialize};
@@ -57,34 +57,39 @@ impl<K: Ord, V> VecMap<K, V> {
/// Add a key value pair to the map.
/// If `key` is less than or equal to the current maximum key
/// the pair will not be added and InvalidKey error will be returned.
pub fn append(&mut self, key: K, value: V) -> Result<(), InvalidKey> {
pub fn append(&mut self, key: K, value: V) -> Result<usize, InvalidKey> {
if let Some((last_key, _last_value)) = self.0.last() {
if &key <= last_key {
return Err(InvalidKey);
}
}
self.0.push((key, value));
Ok(())
let delta_size = self.instrument_vec_op(|vec| vec.push((key, value)));
Ok(delta_size)
}
/// Update the maximum key value pair or add a new key value pair to the map.
/// If `key` is less than the current maximum key no updates or additions
/// will occur and InvalidKey error will be returned.
pub fn append_or_update_last(&mut self, key: K, mut value: V) -> Result<Option<V>, InvalidKey> {
pub fn append_or_update_last(
&mut self,
key: K,
mut value: V,
) -> Result<(Option<V>, usize), InvalidKey> {
if let Some((last_key, last_value)) = self.0.last_mut() {
match key.cmp(last_key) {
Ordering::Less => return Err(InvalidKey),
Ordering::Equal => {
std::mem::swap(last_value, &mut value);
return Ok(Some(value));
const DELTA_SIZE: usize = 0;
return Ok((Some(value), DELTA_SIZE));
}
Ordering::Greater => {}
}
}
self.0.push((key, value));
Ok(None)
let delta_size = self.instrument_vec_op(|vec| vec.push((key, value)));
Ok((None, delta_size))
}
/// Split the map into two.
@@ -110,7 +115,7 @@ impl<K: Ord, V> VecMap<K, V> {
/// Move items from `other` to the end of `self`, leaving `other` empty.
/// If any keys in `other` is less than or equal to any key in `self`,
/// `InvalidKey` error will be returned and no mutation will occur.
pub fn extend(&mut self, other: &mut Self) -> Result<(), InvalidKey> {
pub fn extend(&mut self, other: &mut Self) -> Result<usize, InvalidKey> {
let self_last_opt = self.0.last().map(extract_key);
let other_first_opt = other.0.last().map(extract_key);
@@ -120,9 +125,27 @@ impl<K: Ord, V> VecMap<K, V> {
}
}
self.0.append(&mut other.0);
let delta_size = self.instrument_vec_op(|vec| vec.append(&mut other.0));
Ok(delta_size)
}
Ok(())
/// Instrument an operation on the underlying [`Vec`].
/// Will panic if the operation decreases capacity.
/// Returns the increase in memory usage caused by the op.
fn instrument_vec_op(&mut self, op: impl FnOnce(&mut Vec<(K, V)>)) -> usize {
let old_cap = self.0.capacity();
op(&mut self.0);
let new_cap = self.0.capacity();
match old_cap.cmp(&new_cap) {
Ordering::Less => {
let old_size = Layout::array::<(K, V)>(old_cap).unwrap().size();
let new_size = Layout::array::<(K, V)>(new_cap).unwrap().size();
new_size - old_size
}
Ordering::Equal => 0,
Ordering::Greater => panic!("VecMap capacity shouldn't ever decrease"),
}
}
}