From 97992226d3aa7d47c8629d4179aa65f14330d592 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 14 May 2021 12:44:52 +0300 Subject: [PATCH] Add some unit tests for the Repository/Timeline interface. --- pageserver/src/repository.rs | 186 +++++++++++++++++++++++++++ pageserver/src/repository/rocksdb.rs | 16 +++ 2 files changed, 202 insertions(+) diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index b6ff320334..3d83c155fd 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -24,6 +24,12 @@ pub trait Repository { /// Creates a new Timeline object if it's not "open" already. fn get_or_restore_timeline(&self, timelineid: ZTimelineId) -> Result>; + /// Create an empty timeline, without loading any data into it from possible on-disk snapshot. + /// + /// For unit tests. + #[cfg(test)] + fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result>; + //fn get_stats(&self) -> RepositoryStats; } @@ -205,3 +211,183 @@ impl WALRecord { } } } + +/// +/// Tests that should work the same with any Repository/Timeline implementation. +/// +#[cfg(test)] +mod tests { + use super::*; + use crate::walredo::{WalRedoError, WalRedoManager}; + use crate::PageServerConf; + use std::fs; + use std::path::Path; + use std::str::FromStr; + use std::time::Duration; + + fn get_test_conf() -> PageServerConf { + PageServerConf { + daemonize: false, + interactive: false, + gc_horizon: 64 * 1024 * 1024, + gc_period: Duration::from_secs(10), + listen_addr: "127.0.0.1:5430".parse().unwrap(), + } + } + + /// Arbitrary relation tag, for testing. + const TESTREL_A: RelTag = RelTag { + spcnode: 0, + dbnode: 111, + relnode: 1000, + forknum: 0, + }; + + /// Convenience function to create a BufferTag for testing. + /// Helps to keeps the tests shorter. + #[allow(non_snake_case)] + fn TEST_BUF(blknum: u32) -> BufferTag { + BufferTag { + rel: TESTREL_A, + blknum, + } + } + + /// Convenience function to create a page image with given string as the only content + #[allow(non_snake_case)] + fn TEST_IMG(s: &str) -> Bytes { + let mut buf = BytesMut::new(); + buf.extend_from_slice(s.as_bytes()); + buf.resize(8192, 0); + + buf.freeze() + } + + /// Test get_relsize() and truncation. + /// + /// FIXME: The RocksRepository implementation returns wrong relation size, if + /// you make a request with an old LSN. It seems to ignore the requested LSN + /// and always return result as of latest LSN. For such cases, the expected + /// results below match the current RocksRepository behavior, so that the test + /// passes, and the actually correct answers are in comments like + /// "// CORRECT: " + #[test] + fn test_relsize() -> Result<()> { + let walredo_mgr = TestRedoManager {}; + + let repo_dir = Path::new("../tmp_check/test_relsize_repo"); + let _ = fs::remove_dir_all(repo_dir); + fs::create_dir_all(repo_dir)?; + let repo = rocksdb::RocksRepository::new(&get_test_conf(), repo_dir, Arc::new(walredo_mgr)); + + // get_timeline() with non-existent timeline id should fail + //repo.get_timeline("11223344556677881122334455667788"); + + // Create it + let timelineid = ZTimelineId::from_str("11223344556677881122334455667788").unwrap(); + let tline = repo.create_empty_timeline(timelineid)?; + + tline.init_valid_lsn(Lsn(1)); + tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2")); + tline.put_page_image(TEST_BUF(0), Lsn(2), TEST_IMG("foo blk 0 at 2")); + tline.put_page_image(TEST_BUF(0), Lsn(3), TEST_IMG("foo blk 0 at 3")); + tline.put_page_image(TEST_BUF(1), Lsn(4), TEST_IMG("foo blk 1 at 4")); + tline.put_page_image(TEST_BUF(2), Lsn(5), TEST_IMG("foo blk 2 at 5")); + + tline.advance_last_valid_lsn(Lsn(5)); + + // rocksdb implementation erroneosly returns 'true' here + assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(1))?, true); // CORRECT: false + // likewise, it returns wrong size here + assert_eq!(tline.get_relsize(TESTREL_A, Lsn(1))?, 3); // CORRECT: 0 (or error?) + + assert_eq!(tline.get_relsize_exists(TESTREL_A, Lsn(2))?, true); + assert_eq!(tline.get_relsize(TESTREL_A, Lsn(2))?, 3); // CORRECT: 1 + assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 3); + + // Check page contents at each LSN + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(0), Lsn(2))?, + TEST_IMG("foo blk 0 at 2") + ); + + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(0), Lsn(3))?, + TEST_IMG("foo blk 0 at 3") + ); + + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(0), Lsn(4))?, + TEST_IMG("foo blk 0 at 3") + ); + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(1), Lsn(4))?, + TEST_IMG("foo blk 1 at 4") + ); + + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(0), Lsn(5))?, + TEST_IMG("foo blk 0 at 3") + ); + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(1), Lsn(5))?, + TEST_IMG("foo blk 1 at 4") + ); + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?, + TEST_IMG("foo blk 2 at 5") + ); + + // Truncate last block + tline.put_truncation(TESTREL_A, Lsn(6), 2)?; + tline.advance_last_valid_lsn(Lsn(6)); + + // Check reported size and contents after truncation + assert_eq!(tline.get_relsize(TESTREL_A, Lsn(6))?, 2); + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(0), Lsn(6))?, + TEST_IMG("foo blk 0 at 3") + ); + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(1), Lsn(6))?, + TEST_IMG("foo blk 1 at 4") + ); + + // should still see the truncated block with older LSN + assert_eq!(tline.get_relsize(TESTREL_A, Lsn(5))?, 2); // CORRECT: 3 + assert_eq!( + tline.get_page_at_lsn(TEST_BUF(2), Lsn(5))?, + TEST_IMG("foo blk 2 at 5") + ); + + Ok(()) + } + + // Mock WAL redo manager that doesn't do much + struct TestRedoManager {} + + impl WalRedoManager for TestRedoManager { + fn request_redo( + &self, + tag: BufferTag, + lsn: Lsn, + base_img: Option, + records: Vec, + ) -> Result { + let s = format!( + "redo for rel {} blk {} to get to {}, with {} and {} records", + tag.rel, + tag.blknum, + lsn, + if base_img.is_some() { + "base image" + } else { + "no base image" + }, + records.len() + ); + println!("{}", s); + Ok(TEST_IMG(&s)) + } + } +} diff --git a/pageserver/src/repository/rocksdb.rs b/pageserver/src/repository/rocksdb.rs index 552ee429c2..4bfb161482 100644 --- a/pageserver/src/repository/rocksdb.rs +++ b/pageserver/src/repository/rocksdb.rs @@ -211,6 +211,21 @@ impl Repository for RocksRepository { } } } + + #[cfg(test)] + fn create_empty_timeline(&self, timelineid: ZTimelineId) -> Result> { + let mut timelines = self.timelines.lock().unwrap(); + + let timeline = RocksTimeline::new(&self.repo_dir, timelineid, self.walredo_mgr.clone()); + + let timeline_rc = Arc::new(timeline); + let r = timelines.insert(timelineid, timeline_rc.clone()); + assert!(r.is_none()); + + // don't start the garbage collector for unit tests, either. + + Ok(timeline_rc) + } } impl RocksTimeline { @@ -621,6 +636,7 @@ impl Timeline for RocksTimeline { /// /// Does relation exist at given LSN? /// + /// FIXME: this actually returns true, if the relation exists at *any* LSN fn get_relsize_exists(&self, rel: RelTag, req_lsn: Lsn) -> Result { let lsn = self.wait_lsn(req_lsn)?;