From 0b5b2e8e0b9f789a834d4e9a78b2bc4debc17d26 Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Thu, 7 Jul 2022 14:49:57 +0300 Subject: [PATCH] postgres_ffi/wal_craft: extract trait Crafter Make the intent of the code clearer. --- libs/postgres_ffi/src/xlog_utils.rs | 14 +- .../wal_craft/src/bin/wal_craft.rs | 26 +-- libs/postgres_ffi/wal_craft/src/lib.rs | 207 ++++++++++-------- 3 files changed, 130 insertions(+), 117 deletions(-) diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index b707b10fc8..b7464da887 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -603,9 +603,8 @@ mod tests { .try_init(); } - fn test_end_of_wal( + fn test_end_of_wal( test_name: &str, - generate_wal: impl Fn(&mut postgres::Client) -> anyhow::Result, expected_end_of_wal_non_partial: Lsn, last_segment: &str, ) { @@ -624,7 +623,7 @@ mod tests { cfg.initdb().unwrap(); let mut srv = cfg.start_server().unwrap(); let expected_wal_end: Lsn = - u64::from(generate_wal(&mut srv.connect_with_timeout().unwrap()).unwrap()).into(); + u64::from(C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap()).into(); srv.kill(); // 2. Pick WAL generated by initdb @@ -681,9 +680,8 @@ mod tests { #[test] pub fn test_find_end_of_wal_simple() { init_logging(); - test_end_of_wal( + test_end_of_wal::( "test_find_end_of_wal_simple", - wal_craft::generate_simple, "0/2000000".parse::().unwrap(), "000000010000000000000001", ); @@ -692,9 +690,8 @@ mod tests { #[test] pub fn test_find_end_of_wal_crossing_segment_followed_by_small_one() { init_logging(); - test_end_of_wal( + test_end_of_wal::( "test_find_end_of_wal_crossing_segment_followed_by_small_one", - wal_craft::generate_wal_record_crossing_segment_followed_by_small_one, "0/3000000".parse::().unwrap(), "000000010000000000000002", ); @@ -704,9 +701,8 @@ mod tests { #[ignore = "not yet fixed, needs correct parsing of pre-last segments"] // TODO pub fn test_find_end_of_wal_last_crossing_segment() { init_logging(); - test_end_of_wal( + test_end_of_wal::( "test_find_end_of_wal_last_crossing_segment", - wal_craft::generate_last_wal_record_crossing_segment, "0/3000000".parse::().unwrap(), "000000010000000000000002", ); diff --git a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs index 8297ad4391..2c50bebdd2 100644 --- a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs +++ b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs @@ -10,11 +10,11 @@ fn main() -> Result<()> { .takes_value(true) .help("Type of WAL to craft") .possible_values([ - "simple", - "last_wal_record_xlog_switch", - "last_wal_record_xlog_switch_ends_on_page_boundary", - "last_wal_record_crossing_segment", - "wal_record_crossing_segment_followed_by_small_one", + Simple::NAME, + LastWalRecordXlogSwitch::NAME, + LastWalRecordXlogSwitchEndsOnPageBoundary::NAME, + WalRecordCrossingSegmentFollowedBySmallOne::NAME, + LastWalRecordCrossingSegment::NAME, ]) .required(true); let arg_matches = App::new("Postgres WAL crafter") @@ -56,17 +56,15 @@ fn main() -> Result<()> { let wal_craft = |arg_matches: &ArgMatches, client| { let lsn = match arg_matches.value_of("type").unwrap() { - "simple" => generate_simple(client)?, - "last_wal_record_xlog_switch" => generate_last_wal_record_xlog_switch(client)?, - "last_wal_record_xlog_switch_ends_on_page_boundary" => { - generate_last_wal_record_xlog_switch_ends_on_page_boundary(client)? + Simple::NAME => Simple::craft(client)?, + LastWalRecordXlogSwitch::NAME => LastWalRecordXlogSwitch::craft(client)?, + LastWalRecordXlogSwitchEndsOnPageBoundary::NAME => { + LastWalRecordXlogSwitchEndsOnPageBoundary::craft(client)? } - "last_wal_record_crossing_segment" => { - generate_last_wal_record_crossing_segment(client)? - } - "wal_record_crossing_segment_followed_by_small_one" => { - generate_wal_record_crossing_segment_followed_by_small_one(client)? + WalRecordCrossingSegmentFollowedBySmallOne::NAME => { + WalRecordCrossingSegmentFollowedBySmallOne::craft(client)? } + LastWalRecordCrossingSegment::NAME => LastWalRecordCrossingSegment::craft(client)?, a => panic!("Unknown --type argument: {}", a), }; println!("end_of_wal = {}", lsn); diff --git a/libs/postgres_ffi/wal_craft/src/lib.rs b/libs/postgres_ffi/wal_craft/src/lib.rs index bd87ab2a19..52ecae59ff 100644 --- a/libs/postgres_ffi/wal_craft/src/lib.rs +++ b/libs/postgres_ffi/wal_craft/src/lib.rs @@ -218,6 +218,13 @@ pub fn ensure_server_config(client: &mut impl postgres::GenericClient) -> Result Ok(()) } +pub trait Crafter { + const NAME: &'static str; + + /// Generates WAL using the client `client`. Returns the expected end-of-wal LSN. + fn craft(client: &mut impl postgres::GenericClient) -> Result; +} + fn craft_internal( client: &mut C, f: impl Fn(&mut C, PgLsn) -> Result>, @@ -246,94 +253,102 @@ fn craft_internal( Ok(last_lsn) } -pub fn generate_simple(client: &mut impl postgres::GenericClient) -> Result { - craft_internal(client, |client, _| { - client.execute("CREATE table t(x int)", &[])?; - Ok(None) - }) -} - -pub fn generate_last_wal_record_xlog_switch( - client: &mut impl postgres::GenericClient, -) -> Result { - // Do not use craft_internal because here we end up with flush_lsn exactly on - // the segment boundary and insert_lsn after the initial page header, which is unusual. - ensure_server_config(client)?; - - client.execute("CREATE table t(x int)", &[])?; - let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); - let next_segment = PgLsn::from(0x0200_0000); - ensure!( - after_xlog_switch <= next_segment, - "XLOG_SWITCH message ended after the expected segment boundary: {} > {}", - after_xlog_switch, - next_segment - ); - Ok(next_segment) -} - -pub fn generate_last_wal_record_xlog_switch_ends_on_page_boundary( - client: &mut impl postgres::GenericClient, -) -> Result { - // Do not use craft_internal because here we end up with flush_lsn exactly on - // the segment boundary and insert_lsn after the initial page header, which is unusual. - ensure_server_config(client)?; - - client.execute("CREATE table t(x int)", &[])?; - - // Add padding so the XLOG_SWITCH record ends exactly on XLOG_BLCKSZ boundary. - // We will use logical message as the padding. We start with detecting how much WAL - // it takes for one logical message, considering all alignments and headers. - let base_wal_advance = { - let before_lsn = client.pg_current_wal_insert_lsn()?; - // Small non-empty message bigger than few bytes is more likely than an empty - // message to have the same format as the big padding message. - client.execute( - "SELECT pg_logical_emit_message(false, 'swch', REPEAT('a', 10))", - &[], - )?; - // The XLOG_SWITCH record has no data => its size is exactly XLOG_SIZE_OF_XLOG_RECORD. - (u64::from(client.pg_current_wal_insert_lsn()?) - u64::from(before_lsn)) as usize - + XLOG_SIZE_OF_XLOG_RECORD - }; - let mut remaining_lsn = - XLOG_BLCKSZ - u64::from(client.pg_current_wal_insert_lsn()?) as usize % XLOG_BLCKSZ; - if remaining_lsn < base_wal_advance { - remaining_lsn += XLOG_BLCKSZ; +pub struct Simple; +impl Crafter for Simple { + const NAME: &'static str = "simple"; + fn craft(client: &mut impl postgres::GenericClient) -> Result { + craft_internal(client, |client, _| { + client.execute("CREATE table t(x int)", &[])?; + Ok(None) + }) } - let repeats = 10 + remaining_lsn - base_wal_advance; - info!( - "current_wal_insert_lsn={}, remaining_lsn={}, base_wal_advance={}, repeats={}", - client.pg_current_wal_insert_lsn()?, - remaining_lsn, - base_wal_advance, - repeats - ); - client.execute( - "SELECT pg_logical_emit_message(false, 'swch', REPEAT('a', $1))", - &[&(repeats as i32)], - )?; - info!( - "current_wal_insert_lsn={}, XLOG_SIZE_OF_XLOG_RECORD={}", - client.pg_current_wal_insert_lsn()?, - XLOG_SIZE_OF_XLOG_RECORD - ); +} - // Emit the XLOG_SWITCH - let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); - let next_segment = PgLsn::from(0x0200_0000); - ensure!( - after_xlog_switch < next_segment, - "XLOG_SWITCH message ended on or after the expected segment boundary: {} > {}", - after_xlog_switch, - next_segment - ); - ensure!( - u64::from(after_xlog_switch) as usize % XLOG_BLCKSZ == XLOG_SIZE_OF_XLOG_SHORT_PHD, - "XLOG_SWITCH message ended not on page boundary: {}", - after_xlog_switch - ); - Ok(next_segment) +pub struct LastWalRecordXlogSwitch; +impl Crafter for LastWalRecordXlogSwitch { + const NAME: &'static str = "last_wal_record_xlog_switch"; + fn craft(client: &mut impl postgres::GenericClient) -> Result { + // Do not use generate_internal because here we end up with flush_lsn exactly on + // the segment boundary and insert_lsn after the initial page header, which is unusual. + ensure_server_config(client)?; + + client.execute("CREATE table t(x int)", &[])?; + let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); + let next_segment = PgLsn::from(0x0200_0000); + ensure!( + after_xlog_switch <= next_segment, + "XLOG_SWITCH message ended after the expected segment boundary: {} > {}", + after_xlog_switch, + next_segment + ); + Ok(next_segment) + } +} + +pub struct LastWalRecordXlogSwitchEndsOnPageBoundary; +impl Crafter for LastWalRecordXlogSwitchEndsOnPageBoundary { + const NAME: &'static str = "last_wal_record_xlog_switch_ends_on_page_boundary"; + fn craft(client: &mut impl postgres::GenericClient) -> Result { + // Do not use generate_internal because here we end up with flush_lsn exactly on + // the segment boundary and insert_lsn after the initial page header, which is unusual. + ensure_server_config(client)?; + + client.execute("CREATE table t(x int)", &[])?; + + // Add padding so the XLOG_SWITCH record ends exactly on XLOG_BLCKSZ boundary. + // We will use logical message as the padding. We start with detecting how much WAL + // it takes for one logical message, considering all alignments and headers. + let base_wal_advance = { + let before_lsn = client.pg_current_wal_insert_lsn()?; + // Small non-empty message bigger than few bytes is more likely than an empty + // message to have the same format as the big padding message. + client.execute( + "SELECT pg_logical_emit_message(false, 'swch', REPEAT('a', 10))", + &[], + )?; + // The XLOG_SWITCH record has no data => its size is exactly XLOG_SIZE_OF_XLOG_RECORD. + (u64::from(client.pg_current_wal_insert_lsn()?) - u64::from(before_lsn)) as usize + + XLOG_SIZE_OF_XLOG_RECORD + }; + let mut remaining_lsn = + XLOG_BLCKSZ - u64::from(client.pg_current_wal_insert_lsn()?) as usize % XLOG_BLCKSZ; + if remaining_lsn < base_wal_advance { + remaining_lsn += XLOG_BLCKSZ; + } + let repeats = 10 + remaining_lsn - base_wal_advance; + info!( + "current_wal_insert_lsn={}, remaining_lsn={}, base_wal_advance={}, repeats={}", + client.pg_current_wal_insert_lsn()?, + remaining_lsn, + base_wal_advance, + repeats + ); + client.execute( + "SELECT pg_logical_emit_message(false, 'swch', REPEAT('a', $1))", + &[&(repeats as i32)], + )?; + info!( + "current_wal_insert_lsn={}, XLOG_SIZE_OF_XLOG_RECORD={}", + client.pg_current_wal_insert_lsn()?, + XLOG_SIZE_OF_XLOG_RECORD + ); + + // Emit the XLOG_SWITCH + let after_xlog_switch: PgLsn = client.query_one("SELECT pg_switch_wal()", &[])?.get(0); + let next_segment = PgLsn::from(0x0200_0000); + ensure!( + after_xlog_switch < next_segment, + "XLOG_SWITCH message ended on or after the expected segment boundary: {} > {}", + after_xlog_switch, + next_segment + ); + ensure!( + u64::from(after_xlog_switch) as usize % XLOG_BLCKSZ == XLOG_SIZE_OF_XLOG_SHORT_PHD, + "XLOG_SWITCH message ended not on page boundary: {}", + after_xlog_switch + ); + Ok(next_segment) + } } fn craft_single_logical_message( @@ -378,14 +393,18 @@ fn craft_single_logical_message( }) } -pub fn generate_wal_record_crossing_segment_followed_by_small_one( - client: &mut impl postgres::GenericClient, -) -> Result { - craft_single_logical_message(client, true) +pub struct WalRecordCrossingSegmentFollowedBySmallOne; +impl Crafter for WalRecordCrossingSegmentFollowedBySmallOne { + const NAME: &'static str = "wal_record_crossing_segment_followed_by_small_one"; + fn craft(client: &mut impl postgres::GenericClient) -> Result { + craft_single_logical_message(client, true) + } } -pub fn generate_last_wal_record_crossing_segment( - client: &mut C, -) -> Result { - craft_single_logical_message(client, false) +pub struct LastWalRecordCrossingSegment; +impl Crafter for LastWalRecordCrossingSegment { + const NAME: &'static str = "last_wal_record_crossing_segment"; + fn craft(client: &mut impl postgres::GenericClient) -> Result { + craft_single_logical_message(client, false) + } }