From 19ea486cde7f7261eccf336ccb664ded291b63c7 Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Sat, 9 Jul 2022 14:13:11 +0300 Subject: [PATCH] postgres_ffi/xlog_utils: refactor find_end_of_wal test * Deduce `last_segment` automatically * Get rid of local `wal_dir`/`wal_seg_size` variables * Prepare to test parsing of WAL from multiple specific points, not just the start; extract `check_end_of_wal` function to check both partial and non-partial WAL segments. --- libs/postgres_ffi/src/xlog_utils.rs | 95 +++++++++++++++++--------- libs/postgres_ffi/wal_craft/src/lib.rs | 7 +- 2 files changed, 70 insertions(+), 32 deletions(-) diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index 17891fb94f..b9bd922025 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -15,6 +15,7 @@ use crate::XLogPageHeaderData; use crate::XLogRecord; use crate::XLOG_PAGE_MAGIC; +use crate::pg_constants::WAL_SEGMENT_SIZE; use anyhow::{bail, ensure}; use byteorder::{ByteOrder, LittleEndian}; use bytes::BytesMut; @@ -461,8 +462,7 @@ pub fn find_end_of_wal( pub fn main() { let mut data_dir = PathBuf::new(); data_dir.push("."); - let wal_seg_size = 16 * 1024 * 1024; - let (wal_end, tli) = find_end_of_wal(&data_dir, wal_seg_size, true, Lsn(0)).unwrap(); + let (wal_end, tli) = find_end_of_wal(&data_dir, WAL_SEGMENT_SIZE, true, Lsn(0)).unwrap(); println!( "wal_end={:>08X}{:>08X}, tli={}", (wal_end >> 32) as u32, @@ -606,10 +606,9 @@ mod tests { fn test_end_of_wal( test_name: &str, expected_end_of_wal_non_partial: Lsn, - last_segment: &str, ) { use wal_craft::*; - // 1. Generate some WAL + // Craft some WAL let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("..") .join(".."); @@ -622,24 +621,35 @@ mod tests { } cfg.initdb().unwrap(); let srv = cfg.start_server().unwrap(); - let expected_wal_end: Lsn = + let expected_end_of_wal_partial: Lsn = u64::from(C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap()).into(); srv.kill(); - // 2. Pick WAL generated by initdb - let wal_dir = cfg.datadir.join("pg_wal"); - let wal_seg_size = 16 * 1024 * 1024; - - // 3. Check end_of_wal on non-partial WAL segment (we treat it as fully populated) - let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap(); - let wal_end = Lsn(wal_end); - info!( - "find_end_of_wal returned (wal_end={}, tli={})", - wal_end, tli + // Check find_end_of_wal on the initial WAL + let last_segment = cfg + .wal_dir() + .read_dir() + .unwrap() + .map(|f| f.unwrap().file_name().into_string().unwrap()) + .filter(|fname| IsXLogFileName(fname)) + .max() + .unwrap(); + check_pg_waldump_end_of_wal(&cfg, &last_segment, expected_end_of_wal_partial); + check_end_of_wal( + &cfg, + &last_segment, + Lsn(0), // start from the beginning + expected_end_of_wal_non_partial, + expected_end_of_wal_partial, ); - assert_eq!(wal_end, expected_end_of_wal_non_partial); + } - // 4. Get the actual end of WAL by pg_waldump + fn check_pg_waldump_end_of_wal( + cfg: &wal_craft::Conf, + last_segment: &str, + expected_end_of_wal: Lsn, + ) { + // Get the actual end of WAL by pg_waldump let waldump_output = cfg .pg_waldump("000000010000000000000001", last_segment) .unwrap() @@ -658,32 +668,57 @@ mod tests { let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap(); info!( "waldump erred on {}, expected wal end at {}", - waldump_wal_end, expected_wal_end + waldump_wal_end, expected_end_of_wal ); - assert_eq!(waldump_wal_end, expected_wal_end); + assert_eq!(waldump_wal_end, expected_end_of_wal); + } - // 5. Rename file to partial to actually find last valid lsn - fs::rename( - wal_dir.join(last_segment), - wal_dir.join(format!("{}.partial", last_segment)), - ) - .unwrap(); - let (wal_end, tli) = find_end_of_wal(&wal_dir, wal_seg_size, true, Lsn(0)).unwrap(); + fn check_end_of_wal( + cfg: &wal_craft::Conf, + last_segment: &str, + start_lsn: Lsn, + expected_end_of_wal_non_partial: Lsn, + expected_end_of_wal_partial: Lsn, + ) { + // Check end_of_wal on non-partial WAL segment (we treat it as fully populated) + let (wal_end, tli) = + find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, true, start_lsn).unwrap(); let wal_end = Lsn(wal_end); info!( - "find_end_of_wal returned (wal_end={}, tli={})", + "find_end_of_wal returned (wal_end={}, tli={}) with non-partial WAL segment", wal_end, tli ); - assert_eq!(wal_end, waldump_wal_end); + assert_eq!(wal_end, expected_end_of_wal_non_partial); + + // Rename file to partial to actually find last valid lsn, then rename it back. + fs::rename( + cfg.wal_dir().join(&last_segment), + cfg.wal_dir().join(format!("{}.partial", last_segment)), + ) + .unwrap(); + let (wal_end, tli) = + find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, true, start_lsn).unwrap(); + let wal_end = Lsn(wal_end); + info!( + "find_end_of_wal returned (wal_end={}, tli={}) with partial WAL segment", + wal_end, tli + ); + assert_eq!(wal_end, expected_end_of_wal_partial); + fs::rename( + cfg.wal_dir().join(format!("{}.partial", last_segment)), + cfg.wal_dir().join(last_segment), + ) + .unwrap(); } + const_assert!(WAL_SEGMENT_SIZE == 16 * 1024 * 1024); + #[test] pub fn test_find_end_of_wal_simple() { init_logging(); test_end_of_wal::( "test_find_end_of_wal_simple", "0/2000000".parse::().unwrap(), - "000000010000000000000001", ); } @@ -693,7 +728,6 @@ mod tests { test_end_of_wal::( "test_find_end_of_wal_crossing_segment_followed_by_small_one", "0/3000000".parse::().unwrap(), - "000000010000000000000002", ); } @@ -704,7 +738,6 @@ mod tests { test_end_of_wal::( "test_find_end_of_wal_last_crossing_segment", "0/3000000".parse::().unwrap(), - "000000010000000000000002", ); } diff --git a/libs/postgres_ffi/wal_craft/src/lib.rs b/libs/postgres_ffi/wal_craft/src/lib.rs index 51482137c8..11e62d7fba 100644 --- a/libs/postgres_ffi/wal_craft/src/lib.rs +++ b/libs/postgres_ffi/wal_craft/src/lib.rs @@ -4,6 +4,7 @@ use log::*; use once_cell::sync::Lazy; use postgres::types::PgLsn; use postgres::Client; +use postgres_ffi::pg_constants::WAL_SEGMENT_SIZE; use postgres_ffi::xlog_utils::{ XLOG_BLCKSZ, XLOG_SIZE_OF_XLOG_RECORD, XLOG_SIZE_OF_XLOG_SHORT_PHD, }; @@ -45,6 +46,10 @@ impl Conf { self.pg_distrib_dir.join("lib") } + pub fn wal_dir(&self) -> PathBuf { + self.datadir.join("pg_wal") + } + fn new_pg_command(&self, command: impl AsRef) -> Result { let path = self.pg_bin_dir().join(command); ensure!(path.exists(), "Command {:?} does not exist", path); @@ -211,7 +216,7 @@ pub fn ensure_server_config(client: &mut impl postgres::GenericClient) -> Result "Unexpected wal_segment_size unit" ); ensure!( - wal_segment_size.get::<_, i64>("setting") == 16 * 1024 * 1024, + wal_segment_size.get::<_, i64>("setting") == WAL_SEGMENT_SIZE as i64, "Unexpected wal_segment_size in bytes" );