From 94003e1ebc4f1699e4ecdc4d0dc59985219b6e6f Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Sat, 9 Jul 2022 14:37:08 +0300 Subject: [PATCH] postgres_ffi: test restoring from intermediate LSNs by wal_craft --- libs/postgres_ffi/src/xlog_utils.rs | 54 +++++++++++++++---- .../wal_craft/src/bin/wal_craft.rs | 7 ++- libs/postgres_ffi/wal_craft/src/lib.rs | 43 +++++++++------ test_runner/fixtures/neon_fixtures.py | 5 +- 4 files changed, 77 insertions(+), 32 deletions(-) diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index b9bd922025..520870cc53 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -621,8 +621,13 @@ mod tests { } cfg.initdb().unwrap(); let srv = cfg.start_server().unwrap(); - let expected_end_of_wal_partial: Lsn = - u64::from(C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap()).into(); + let (intermediate_lsns, expected_end_of_wal_partial) = + C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap(); + let intermediate_lsns: Vec = intermediate_lsns + .iter() + .map(|&lsn| u64::from(lsn).into()) + .collect(); + let expected_end_of_wal_partial: Lsn = u64::from(expected_end_of_wal_partial).into(); srv.kill(); // Check find_end_of_wal on the initial WAL @@ -635,13 +640,44 @@ mod tests { .max() .unwrap(); check_pg_waldump_end_of_wal(&cfg, &last_segment, expected_end_of_wal_partial); - check_end_of_wal( - &cfg, - &last_segment, - Lsn(0), // start from the beginning - expected_end_of_wal_non_partial, - expected_end_of_wal_partial, - ); + for start_lsn in std::iter::once(Lsn(0)) + .chain(intermediate_lsns) + .chain(std::iter::once(expected_end_of_wal_partial)) + { + // Erase all WAL before `start_lsn` to ensure it's not used by `find_end_of_wal`. + // We assume that `start_lsn` is non-decreasing. + info!( + "Checking with start_lsn={}, erasing WAL before it", + start_lsn + ); + for file in fs::read_dir(cfg.wal_dir()).unwrap().flatten() { + let fname = file.file_name().into_string().unwrap(); + if !IsXLogFileName(&fname) { + continue; + } + let (segno, _) = XLogFromFileName(&fname, WAL_SEGMENT_SIZE); + let seg_start_lsn = XLogSegNoOffsetToRecPtr(segno, 0, WAL_SEGMENT_SIZE); + if seg_start_lsn > u64::from(start_lsn) { + continue; + } + let mut f = File::options().write(true).open(file.path()).unwrap(); + const ZEROS: [u8; WAL_SEGMENT_SIZE] = [0u8; WAL_SEGMENT_SIZE]; + f.write_all( + &ZEROS[0..min( + WAL_SEGMENT_SIZE, + (u64::from(start_lsn) - seg_start_lsn) as usize, + )], + ) + .unwrap(); + } + check_end_of_wal( + &cfg, + &last_segment, + start_lsn, + expected_end_of_wal_non_partial, + expected_end_of_wal_partial, + ); + } } fn check_pg_waldump_end_of_wal( diff --git a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs index 13892538d0..938f8f421b 100644 --- a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs +++ b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs @@ -55,7 +55,7 @@ fn main() -> Result<()> { .get_matches(); let wal_craft = |arg_matches: &ArgMatches, client| { - let lsn = match arg_matches.value_of("type").unwrap() { + let (intermediate_lsns, end_of_wal_lsn) = match arg_matches.value_of("type").unwrap() { Simple::NAME => Simple::craft(client)?, LastWalRecordXlogSwitch::NAME => LastWalRecordXlogSwitch::craft(client)?, LastWalRecordXlogSwitchEndsOnPageBoundary::NAME => { @@ -67,7 +67,10 @@ fn main() -> Result<()> { LastWalRecordCrossingSegment::NAME => LastWalRecordCrossingSegment::craft(client)?, a => panic!("Unknown --type argument: {}", a), }; - println!("end_of_wal = {}", lsn); + for lsn in intermediate_lsns { + println!("intermediate_lsn = {}", lsn); + } + println!("end_of_wal = {}", end_of_wal_lsn); Ok(()) }; diff --git a/libs/postgres_ffi/wal_craft/src/lib.rs b/libs/postgres_ffi/wal_craft/src/lib.rs index 11e62d7fba..e3b666da41 100644 --- a/libs/postgres_ffi/wal_craft/src/lib.rs +++ b/libs/postgres_ffi/wal_craft/src/lib.rs @@ -226,20 +226,24 @@ pub fn ensure_server_config(client: &mut impl postgres::GenericClient) -> Result pub trait Crafter { const NAME: &'static str; - /// Generates WAL using the client `client`. Returns the expected end-of-wal LSN. - fn craft(client: &mut impl postgres::GenericClient) -> Result; + /// Generates WAL using the client `client`. Returns a pair of: + /// * A vector of some valid "interesting" intermediate LSNs which one may start reading from. + /// May include or exclude Lsn(0) and the end-of-wal. + /// * The expected end-of-wal LSN. + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)>; } fn craft_internal( client: &mut C, - f: impl Fn(&mut C, PgLsn) -> Result>, -) -> Result { + f: impl Fn(&mut C, PgLsn) -> Result<(Vec, Option)>, +) -> Result<(Vec, PgLsn)> { ensure_server_config(client)?; let initial_lsn = client.pg_current_wal_insert_lsn()?; info!("LSN initial = {}", initial_lsn); - let last_lsn = match f(client, initial_lsn)? { + let (mut intermediate_lsns, last_lsn) = f(client, initial_lsn)?; + let last_lsn = match last_lsn { None => client.pg_current_wal_insert_lsn()?, Some(last_lsn) => match last_lsn.cmp(&client.pg_current_wal_insert_lsn()?) { Ordering::Less => bail!("Some records were inserted after the crafted WAL"), @@ -247,6 +251,9 @@ fn craft_internal( Ordering::Greater => bail!("Reported LSN is greater than insert_lsn"), }, }; + if !intermediate_lsns.starts_with(&[initial_lsn]) { + intermediate_lsns.insert(0, initial_lsn); + } // Some records may be not flushed, e.g. non-transactional logical messages. client.execute("select neon_xlogflush(pg_current_wal_insert_lsn())", &[])?; @@ -255,16 +262,16 @@ fn craft_internal( Ordering::Equal => {} Ordering::Greater => bail!("Reported LSN is greater than flush_lsn"), } - Ok(last_lsn) + Ok((intermediate_lsns, last_lsn)) } pub struct Simple; impl Crafter for Simple { const NAME: &'static str = "simple"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { craft_internal(client, |client, _| { client.execute("CREATE table t(x int)", &[])?; - Ok(None) + Ok((Vec::new(), None)) }) } } @@ -272,12 +279,13 @@ impl Crafter for Simple { pub struct LastWalRecordXlogSwitch; impl Crafter for LastWalRecordXlogSwitch { const NAME: &'static str = "last_wal_record_xlog_switch"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { // Do not use generate_internal because here we end up with flush_lsn exactly on // the segment boundary and insert_lsn after the initial page header, which is unusual. ensure_server_config(client)?; client.execute("CREATE table t(x int)", &[])?; + let before_xlog_switch = client.pg_current_wal_insert_lsn()?; let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); let next_segment = PgLsn::from(0x0200_0000); ensure!( @@ -286,14 +294,14 @@ impl Crafter for LastWalRecordXlogSwitch { after_xlog_switch, next_segment ); - Ok(next_segment) + Ok((vec![before_xlog_switch, after_xlog_switch], next_segment)) } } pub struct LastWalRecordXlogSwitchEndsOnPageBoundary; impl Crafter for LastWalRecordXlogSwitchEndsOnPageBoundary { const NAME: &'static str = "last_wal_record_xlog_switch_ends_on_page_boundary"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { // Do not use generate_internal because here we end up with flush_lsn exactly on // the segment boundary and insert_lsn after the initial page header, which is unusual. ensure_server_config(client)?; @@ -339,6 +347,7 @@ impl Crafter for LastWalRecordXlogSwitchEndsOnPageBoundary { ); // Emit the XLOG_SWITCH + let before_xlog_switch = client.pg_current_wal_insert_lsn()?; let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); let next_segment = PgLsn::from(0x0200_0000); ensure!( @@ -352,14 +361,14 @@ impl Crafter for LastWalRecordXlogSwitchEndsOnPageBoundary { "XLOG_SWITCH message ended not on page boundary: {}", after_xlog_switch ); - Ok(next_segment) + Ok((vec![before_xlog_switch, after_xlog_switch], next_segment)) } } fn craft_single_logical_message( client: &mut impl postgres::GenericClient, transactional: bool, -) -> Result { +) -> Result<(Vec, PgLsn)> { craft_internal(client, |client, initial_lsn| { ensure!( initial_lsn < PgLsn::from(0x0200_0000 - 1024 * 1024), @@ -391,9 +400,9 @@ fn craft_single_logical_message( message_lsn < after_message_lsn, "No record found after the emitted message" ); - Ok(Some(after_message_lsn)) + Ok((vec![message_lsn], Some(after_message_lsn))) } else { - Ok(Some(message_lsn)) + Ok((Vec::new(), Some(message_lsn))) } }) } @@ -401,7 +410,7 @@ fn craft_single_logical_message( pub struct WalRecordCrossingSegmentFollowedBySmallOne; impl Crafter for WalRecordCrossingSegmentFollowedBySmallOne { const NAME: &'static str = "wal_record_crossing_segment_followed_by_small_one"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { craft_single_logical_message(client, true) } } @@ -409,7 +418,7 @@ impl Crafter for WalRecordCrossingSegmentFollowedBySmallOne { pub struct LastWalRecordCrossingSegment; impl Crafter for LastWalRecordCrossingSegment { const NAME: &'static str = "last_wal_record_crossing_segment"; - fn craft(client: &mut impl postgres::GenericClient) -> Result { + fn craft(client: &mut impl postgres::GenericClient) -> Result<(Vec, PgLsn)> { craft_single_logical_message(client, false) } } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index e2bf7da79d..3a6a233208 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1276,12 +1276,9 @@ class WalCraft(AbstractNeonCli): res.check_returncode() return res.stdout.split('\n') - def in_existing(self, type: str, connection: str) -> int: + def in_existing(self, type: str, connection: str) -> None: res = self.raw_cli(["in-existing", type, connection]) res.check_returncode() - m = re.fullmatch(r'end_of_wal = (.*)\n', res.stdout) - assert m - return lsn_from_hex(m.group(1)) class NeonPageserver(PgProtocol):