rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-08 22:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

list_rels() and list_nonrels() refactoring:

Browse Source 
move shared code to list_relishes() function.
This commit is contained in:
anastasia
2021-09-07 15:49:39 +03:00
committed by lubennikovaav
parent 674807eee1
commit b79754d06e
4 changed files with 135 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
pageserver/src/layered_repository.rs
View File

@@ -18,6 +18,7 @@ use log::*;
use postgres_ffi::pg_constants::BLCKSZ;
use serde::{Deserialize, Serialize};
use std::collections::hash_map::Entry;
use std::collections::HashMap;
use std::collections::{BTreeSet, HashSet};
use std::fs::File;
@@ -645,89 +646,62 @@ impl Timeline for LayeredTimeline {
Ok(result)
}
fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>> {
trace!("list_rels called at {}", lsn);
fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelishTag>> {
let request_tag = RelTag {
spcnode: spcnode,
dbnode: dbnode,
relnode: 0,
forknum: 0,
};
// List all rels in this timeline, and all its ancestors.
let mut all_rels_map: HashMap<RelTag, bool> = HashMap::new();
let mut result = HashSet::new();
let mut timeline = self;
loop {
timeline.layers.lock().unwrap().dump()?;
let rels = timeline
.layers
.lock()
.unwrap()
.list_rels(spcnode, dbnode, lsn)?;
for (&new_rel, &new_rel_exists) in rels.iter() {
if let Some(rel_exists) = all_rels_map.get(&new_rel) {
trace!(
"list_rels() Newer version of the object {} is already found: exists {}",
new_rel,
rel_exists
);
} else {
all_rels_map.insert(new_rel, new_rel_exists);
trace!(
"list_rels() Newer version of the object {} NOT found: exists {}",
new_rel,
new_rel_exists
);
}
}
if let Some(ancestor) = timeline.ancestor_timeline.as_ref() {
timeline = ancestor;
continue;
} else {
break;
}
}
// Filter out dropped relations
for (&new_rel, &new_rel_exists) in all_rels_map.iter() {
if new_rel_exists {
result.insert(new_rel);
trace!("list_rels() List object {}", new_rel);
} else {
trace!("list_rels() Filter out droped object {}", new_rel);
}
}
Ok(result)
self.list_relishes(Some(request_tag), lsn)
}
fn list_nonrels(&self, lsn: Lsn) -> Result<HashSet<RelishTag>> {
info!("list_nonrels called at {}", lsn);
let lsn = self.wait_lsn(lsn)?;
// List all nonrels in this timeline, and all its ancestors.
let mut all_rels_map: HashMap<RelishTag, bool> = HashMap::new();
self.list_relishes(None, lsn)
}
fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashSet<RelishTag>> {
trace!("list_relishes called at {}", lsn);
// List of all relishes along with a flag that marks if they exist at the given lsn.
let mut all_relishes_map: HashMap<RelishTag, bool> = HashMap::new();
let mut result = HashSet::new();
let mut timeline = self;
loop {
let rels = timeline.layers.lock().unwrap().list_nonrels(lsn)?;
for (&new_rel, &new_rel_exists) in rels.iter() {
if let Some(rel_exists) = all_rels_map.get(&new_rel) {
trace!(
"list_nonrels() Newer version of the object {} is already found: exists {}",
new_rel,
rel_exists
);
} else {
all_rels_map.insert(new_rel, new_rel_exists);
trace!(
"list_nonrels() Newer version of the object {} NOT found: exists {}",
new_rel,
new_rel_exists
);
// Iterate through layers back in time and find the most
// recent state of the relish. Don't add relish to the list
// if newer version is already there.
//
// This most recent version can represent dropped or existing relish.
// We will filter dropped relishes below.
//
loop {
let rels = timeline.layers.lock().unwrap().list_relishes(tag, lsn)?;
for (&new_relish, &new_relish_exists) in rels.iter() {
match all_relishes_map.entry(new_relish) {
Entry::Occupied(o) => {
trace!(
"Newer version of the object {} is already found: exists {}",
new_relish,
o.get(),
);
}
Entry::Vacant(v) => {
v.insert(new_relish_exists);
trace!(
"Newer version of the object {} NOT found. Insert NEW: exists {}",
new_relish,
new_relish_exists
);
}
}
}
if let Some(ancestor) = timeline.ancestor_timeline.as_ref() {
timeline = ancestor;
continue;
@@ -736,13 +710,13 @@ impl Timeline for LayeredTimeline {
}
}
// Filter out dropped relations
for (&new_rel, &new_rel_exists) in all_rels_map.iter() {
if new_rel_exists {
result.insert(new_rel);
trace!("list_nonrels() List object {}", new_rel);
// Filter out dropped relishes
for (&new_relish, &new_relish_exists) in all_relishes_map.iter() {
if new_relish_exists {
result.insert(new_relish);
trace!("List object {}", new_relish);
} else {
trace!("list_nonrels() Filter out droped object {}", new_rel);
trace!("Filter out droped object {}", new_relish);
}
}
@@ -921,7 +895,7 @@ impl Timeline for LayeredTimeline {
let all_rels = self.list_rels(0, 0, lsn)?;
for rel in all_rels {
if let Some(size) = self.get_relish_size(RelishTag::Relation(rel), lsn)? {
if let Some(size) = self.get_relish_size(rel, lsn)? {
total_blocks += size as usize;
}
}

82
pageserver/src/layered_repository/layer_map.rs
View File

@@ -14,10 +14,8 @@ use crate::layered_repository::InMemoryLayer;
use crate::relish::*;
use anyhow::Result;
use lazy_static::lazy_static;
use log::*;
use std::cmp::Ordering;
use std::collections::{BTreeMap, BinaryHeap, HashMap};
use std::ops::Bound::Included;
use std::sync::Arc;
use zenith_metrics::{register_int_gauge, IntGauge};
use zenith_utils::lsn::Lsn;
@@ -144,56 +142,29 @@ impl LayerMap {
// List relations along with a flag that marks if they exist at the given lsn.
// spcnode 0 and dbnode 0 have special meanings and mean all tabespaces/databases.
pub fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashMap<RelTag, bool>> {
let mut rels: HashMap<RelTag, bool> = HashMap::new();
for (seg, segentry) in self.segs.iter() {
if let RelishTag::Relation(reltag) = seg.rel {
if (spcnode == 0 || reltag.spcnode == spcnode)
&& (dbnode == 0 || reltag.dbnode == dbnode)
{
if let Some(layer) = &segentry.open {
if layer.get_start_lsn() <= lsn && lsn <= layer.get_end_lsn() {
let exists = layer.get_seg_exists(lsn)?;
trace!("Open layer list object {}: exists {}", reltag, exists);
rels.insert(reltag, exists);
}
} else if let Some((_, layer)) = segentry
.historic
.range((Included(Lsn(0)), Included(lsn)))
.next_back()
{
let exists = layer.get_seg_exists(lsn)?;
rels.insert(reltag, exists);
}
}
}
}
Ok(rels)
}
// List non-relation relishes that exist at the lsn
pub fn list_nonrels(&self, lsn: Lsn) -> Result<HashMap<RelishTag, bool>> {
// Pass Tag if we're only interested in some relations.
pub fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashMap<RelishTag, bool>> {
let mut rels: HashMap<RelishTag, bool> = HashMap::new();
// Scan the timeline directory to get all rels in this timeline.
for (seg, segentry) in self.segs.iter() {
if let RelishTag::Relation(_) = seg.rel {
} else {
// Add only if it exists at the requested LSN.
if let Some(layer) = &segentry.open {
if layer.get_start_lsn() <= lsn && lsn <= layer.get_end_lsn() {
let exists = layer.get_seg_exists(lsn)?;
trace!("Open layer list object {}: exists {}", seg.rel, exists);
rels.insert(seg.rel, exists);
match seg.rel {
RelishTag::Relation(reltag) => {
if let Some(request_rel) = tag {
if (request_rel.spcnode == 0 || reltag.spcnode == request_rel.spcnode)
&& (request_rel.dbnode == 0 || reltag.dbnode == request_rel.dbnode)
{
if let Some(exists) = segentry.exists_at_lsn(lsn) {
rels.insert(seg.rel, exists);
}
}
}
}
_ => {
if tag == None {
if let Some(exists) = segentry.exists_at_lsn(lsn) {
rels.insert(seg.rel, exists);
}
}
} else if let Some((_k, layer)) = segentry
.historic
.range((Included(Lsn(0)), Included(lsn)))
.next_back()
{
let exists = layer.get_seg_exists(lsn)?;
rels.insert(seg.rel, exists);
}
}
}
@@ -277,6 +248,21 @@ impl Default for SegEntry {
}
impl SegEntry {
/// Does the segment exist at given LSN?
/// Return None if object is not found in this SegEntry.
fn exists_at_lsn(&self, lsn: Lsn) -> Option<bool> {
if let Some(layer) = &self.open {
if layer.get_start_lsn() <= lsn && lsn <= layer.get_end_lsn() {
let exists = layer.get_seg_exists(lsn).ok()?;
return Some(exists);
}
} else if let Some((_, layer)) = self.historic.range(..=lsn).next_back() {
let exists = layer.get_seg_exists(lsn).ok()?;
return Some(exists);
}
None
}
pub fn get(&self, lsn: Lsn) -> Option<Arc<dyn Layer>> {
if let Some(open) = &self.open {
if open.get_start_lsn() <= lsn {

38
pageserver/src/repository.rs
View File

@@ -106,10 +106,14 @@ pub trait Timeline: Send + Sync {
/// Does relation exist?
fn get_rel_exists(&self, tag: RelishTag, lsn: Lsn) -> Result<bool>;
/// Get a list of all distinct relations in given tablespace and database.
fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelTag>>;
/// Get a list of all existing relations
/// Pass RelTag to get relation objects or None to get nonrels.
fn list_relishes(&self, tag: Option<RelTag>, lsn: Lsn) -> Result<HashSet<RelishTag>>;
/// Get a list of non-relational objects
/// Get a list of all existing relations in given tablespace and database.
fn list_rels(&self, spcnode: u32, dbnode: u32, lsn: Lsn) -> Result<HashSet<RelishTag>>;
/// Get a list of all existing non-relational objects
fn list_nonrels(&self, lsn: Lsn) -> Result<HashSet<RelishTag>>;
//------------------------------------------------------------------------------
@@ -480,33 +484,37 @@ mod tests {
// Create a relation on the timeline
tline.put_page_image(TESTREL_A, 0, Lsn(0x20), TEST_IMG("foo blk 0 at 2"))?;
// Check that list_rels() lists it after LSN 2, but no before it.
let reltag = match TESTREL_A {
RelishTag::Relation(reltag) => reltag,
_ => panic!("unexpected relish"),
};
assert!(!tline.list_rels(0, TESTDB, Lsn(0x10))?.contains(&reltag));
assert!(tline.list_rels(0, TESTDB, Lsn(0x20))?.contains(&reltag));
assert!(tline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&reltag));
// Check that list_rels() lists it after LSN 2, but no before it
assert!(!tline.list_rels(0, TESTDB, Lsn(0x10))?.contains(&TESTREL_A));
assert!(tline.list_rels(0, TESTDB, Lsn(0x20))?.contains(&TESTREL_A));
assert!(tline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&TESTREL_A));
// Create a branch, check that the relation is visible there
let newtimelineid = ZTimelineId::from_str("AA223344556677881122334455667788").unwrap();
repo.branch_timeline(timelineid, newtimelineid, Lsn(0x30))?;
let newtline = repo.get_timeline(newtimelineid)?;
assert!(newtline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&reltag));
assert!(newtline
.list_rels(0, TESTDB, Lsn(0x30))?
.contains(&TESTREL_A));
// Drop it on the branch
newtline.drop_relish(TESTREL_A, Lsn(0x40))?;
// Check that it's no longer listed on the branch after the point where it was dropped
assert!(newtline.list_rels(0, TESTDB, Lsn(0x30))?.contains(&reltag));
assert!(!newtline.list_rels(0, TESTDB, Lsn(0x40))?.contains(&reltag));
assert!(newtline
.list_rels(0, TESTDB, Lsn(0x30))?
.contains(&TESTREL_A));
assert!(!newtline
.list_rels(0, TESTDB, Lsn(0x40))?
.contains(&TESTREL_A));
// Run checkpoint and garbage collection and check that it's still not visible
newtline.checkpoint()?;
repo.gc_iteration(Some(newtimelineid), 0, true)?;
assert!(!newtline.list_rels(0, TESTDB, Lsn(0x40))?.contains(&reltag));
assert!(!newtline
.list_rels(0, TESTDB, Lsn(0x40))?
.contains(&TESTREL_A));
Ok(())
}

53
pageserver/src/restore_local_repo.rs
View File

@@ -395,7 +395,7 @@ pub fn save_decoded_record(
for tablespace_id in dropdb.tablespace_ids {
let rels = timeline.list_rels(tablespace_id, dropdb.db_id, lsn)?;
for rel in rels {
timeline.drop_relish(RelishTag::Relation(rel), lsn)?;
timeline.drop_relish(rel, lsn)?;
}
trace!(
"Drop FileNodeMap {}, {} at lsn {}",
@@ -558,37 +558,36 @@ fn save_xlog_dbase_create(timeline: &dyn Timeline, lsn: Lsn, rec: &XlCreateDatab
let mut num_rels_copied = 0;
let mut num_blocks_copied = 0;
for src_rel in rels {
assert_eq!(src_rel.spcnode, src_tablespace_id);
assert_eq!(src_rel.dbnode, src_db_id);
for rel in rels {
if let RelishTag::Relation(src_rel) = rel {
assert_eq!(src_rel.spcnode, src_tablespace_id);
assert_eq!(src_rel.dbnode, src_db_id);
let nblocks = timeline
.get_relish_size(RelishTag::Relation(src_rel), req_lsn)?
.unwrap_or(0);
let dst_rel = RelTag {
spcnode: tablespace_id,
dbnode: db_id,
relnode: src_rel.relnode,
forknum: src_rel.forknum,
};
let nblocks = timeline.get_relish_size(rel, req_lsn)?.unwrap_or(0);
let dst_rel = RelTag {
spcnode: tablespace_id,
dbnode: db_id,
relnode: src_rel.relnode,
forknum: src_rel.forknum,
};
// Copy content
for blknum in 0..nblocks {
let content =
timeline.get_page_at_lsn_nowait(RelishTag::Relation(src_rel), blknum, req_lsn)?;
// Copy content
for blknum in 0..nblocks {
let content = timeline.get_page_at_lsn_nowait(rel, blknum, req_lsn)?;
debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);
debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);
timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content)?;
num_blocks_copied += 1;
timeline.put_page_image(RelishTag::Relation(dst_rel), blknum, lsn, content)?;
num_blocks_copied += 1;
}
if nblocks == 0 {
// make sure we have some trace of the relation, even if it's empty
timeline.put_truncation(RelishTag::Relation(dst_rel), lsn, 0)?;
}
num_rels_copied += 1;
}
if nblocks == 0 {
// make sure we have some trace of the relation, even if it's empty
timeline.put_truncation(RelishTag::Relation(dst_rel), lsn, 0)?;
}
num_rels_copied += 1;
}
// Copy relfilemap