From 60e5dc10e6df9e768efc7a982b20b2140316750e Mon Sep 17 00:00:00 2001 From: Egor Suvorov Date: Fri, 8 Jul 2022 15:19:53 +0300 Subject: [PATCH] postgres_ffi/wal_generate: use 'craft' instead of 'generate' It does very fine-tuned byte-to-byte WAL crafting, not a sloppy generation. Hence 'craft' sounds like a better description. --- Cargo.lock | 4 ++-- libs/postgres_ffi/Cargo.toml | 2 +- libs/postgres_ffi/src/xlog_utils.rs | 10 ++++---- .../{wal_generate => wal_craft}/Cargo.toml | 2 +- .../src/bin/wal_craft.rs} | 24 +++++++++---------- .../{wal_generate => wal_craft}/src/lib.rs | 20 ++++++++-------- .../batch_others/test_crafted_wal_end.py | 8 +++---- test_runner/fixtures/neon_fixtures.py | 6 ++--- 8 files changed, 37 insertions(+), 39 deletions(-) rename libs/postgres_ffi/{wal_generate => wal_craft}/Cargo.toml (94%) rename libs/postgres_ffi/{wal_generate/src/bin/wal_generate.rs => wal_craft/src/bin/wal_craft.rs} (81%) rename libs/postgres_ffi/{wal_generate => wal_craft}/src/lib.rs (96%) diff --git a/Cargo.lock b/Cargo.lock index 6924c0c74a..4f1d727ae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2151,7 +2151,7 @@ dependencies = [ "serde", "thiserror", "utils", - "wal_generate", + "wal_craft", "workspace_hack", ] @@ -3753,7 +3753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] -name = "wal_generate" +name = "wal_craft" version = "0.1.0" dependencies = [ "anyhow", diff --git a/libs/postgres_ffi/Cargo.toml b/libs/postgres_ffi/Cargo.toml index 129c93cf6d..c9cc858ab9 100644 --- a/libs/postgres_ffi/Cargo.toml +++ b/libs/postgres_ffi/Cargo.toml @@ -23,7 +23,7 @@ workspace_hack = { version = "0.1", path = "../../workspace_hack" } [dev-dependencies] env_logger = "0.9" postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" } -wal_generate = { path = "wal_generate" } +wal_craft = { path = "wal_craft" } [build-dependencies] bindgen = "0.59.1" diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index 67541d844e..b707b10fc8 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -597,7 +597,7 @@ mod tests { fn init_logging() { let _ = env_logger::Builder::from_env( env_logger::Env::default() - .default_filter_or("wal_generate=info,postgres_ffi::xlog_utils=trace"), + .default_filter_or("wal_craft=info,postgres_ffi::xlog_utils=trace"), ) .is_test(true) .try_init(); @@ -609,7 +609,7 @@ mod tests { expected_end_of_wal_non_partial: Lsn, last_segment: &str, ) { - use wal_generate::*; + use wal_craft::*; // 1. Generate some WAL let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("..") @@ -683,7 +683,7 @@ mod tests { init_logging(); test_end_of_wal( "test_find_end_of_wal_simple", - wal_generate::generate_simple, + wal_craft::generate_simple, "0/2000000".parse::().unwrap(), "000000010000000000000001", ); @@ -694,7 +694,7 @@ mod tests { init_logging(); test_end_of_wal( "test_find_end_of_wal_crossing_segment_followed_by_small_one", - wal_generate::generate_wal_record_crossing_segment_followed_by_small_one, + wal_craft::generate_wal_record_crossing_segment_followed_by_small_one, "0/3000000".parse::().unwrap(), "000000010000000000000002", ); @@ -706,7 +706,7 @@ mod tests { init_logging(); test_end_of_wal( "test_find_end_of_wal_last_crossing_segment", - wal_generate::generate_last_wal_record_crossing_segment, + wal_craft::generate_last_wal_record_crossing_segment, "0/3000000".parse::().unwrap(), "000000010000000000000002", ); diff --git a/libs/postgres_ffi/wal_generate/Cargo.toml b/libs/postgres_ffi/wal_craft/Cargo.toml similarity index 94% rename from libs/postgres_ffi/wal_generate/Cargo.toml rename to libs/postgres_ffi/wal_craft/Cargo.toml index ce1a60c4f8..374c8e2e55 100644 --- a/libs/postgres_ffi/wal_generate/Cargo.toml +++ b/libs/postgres_ffi/wal_craft/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "wal_generate" +name = "wal_craft" version = "0.1.0" edition = "2021" diff --git a/libs/postgres_ffi/wal_generate/src/bin/wal_generate.rs b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs similarity index 81% rename from libs/postgres_ffi/wal_generate/src/bin/wal_generate.rs rename to libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs index 1549bfb505..8297ad4391 100644 --- a/libs/postgres_ffi/wal_generate/src/bin/wal_generate.rs +++ b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs @@ -1,16 +1,14 @@ use anyhow::*; use clap::{App, Arg, ArgMatches}; use std::str::FromStr; -use wal_generate::*; +use wal_craft::*; fn main() -> Result<()> { - env_logger::Builder::from_env( - env_logger::Env::default().default_filter_or("wal_generate=info"), - ) - .init(); + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("wal_craft=info")) + .init(); let type_arg = &Arg::new("type") .takes_value(true) - .help("Type of WAL to generate") + .help("Type of WAL to craft") .possible_values([ "simple", "last_wal_record_xlog_switch", @@ -19,15 +17,15 @@ fn main() -> Result<()> { "wal_record_crossing_segment_followed_by_small_one", ]) .required(true); - let arg_matches = App::new("Postgres WAL generator") - .about("Generates Postgres databases with specific WAL properties") + let arg_matches = App::new("Postgres WAL crafter") + .about("Crafts Postgres databases with specific WAL properties") .subcommand( App::new("print-postgres-config") .about("Print the configuration required for PostgreSQL server before running this script") ) .subcommand( App::new("with-initdb") - .about("Generate WAL in a new data directory first initialized with initdb") + .about("Craft WAL in a new data directory first initialized with initdb") .arg(type_arg) .arg( Arg::new("datadir") @@ -45,7 +43,7 @@ fn main() -> Result<()> { ) .subcommand( App::new("in-existing") - .about("Generate WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.") + .about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.") .arg(type_arg) .arg( Arg::new("connection") @@ -56,7 +54,7 @@ fn main() -> Result<()> { ) .get_matches(); - let wal_generate = |arg_matches: &ArgMatches, client| { + let wal_craft = |arg_matches: &ArgMatches, client| { let lsn = match arg_matches.value_of("type").unwrap() { "simple" => generate_simple(client)?, "last_wal_record_xlog_switch" => generate_last_wal_record_xlog_switch(client)?, @@ -90,11 +88,11 @@ fn main() -> Result<()> { }; cfg.initdb()?; let mut srv = cfg.start_server()?; - wal_generate(arg_matches, &mut srv.connect_with_timeout()?)?; + wal_craft(arg_matches, &mut srv.connect_with_timeout()?)?; srv.kill(); Ok(()) } - Some(("in-existing", arg_matches)) => wal_generate( + Some(("in-existing", arg_matches)) => wal_craft( arg_matches, &mut postgres::Config::from_str(arg_matches.value_of("connection").unwrap())? .connect(postgres::NoTls)?, diff --git a/libs/postgres_ffi/wal_generate/src/lib.rs b/libs/postgres_ffi/wal_craft/src/lib.rs similarity index 96% rename from libs/postgres_ffi/wal_generate/src/lib.rs rename to libs/postgres_ffi/wal_craft/src/lib.rs index ac6fcc441d..bd87ab2a19 100644 --- a/libs/postgres_ffi/wal_generate/src/lib.rs +++ b/libs/postgres_ffi/wal_craft/src/lib.rs @@ -218,7 +218,7 @@ pub fn ensure_server_config(client: &mut impl postgres::GenericClient) -> Result Ok(()) } -fn generate_internal( +fn craft_internal( client: &mut C, f: impl Fn(&mut C, PgLsn) -> Result>, ) -> Result { @@ -230,7 +230,7 @@ fn generate_internal( let last_lsn = match f(client, initial_lsn)? { None => client.pg_current_wal_insert_lsn()?, Some(last_lsn) => match last_lsn.cmp(&client.pg_current_wal_insert_lsn()?) { - Ordering::Less => bail!("Some records were inserted after the generated WAL"), + Ordering::Less => bail!("Some records were inserted after the crafted WAL"), Ordering::Equal => last_lsn, Ordering::Greater => bail!("Reported LSN is greater than insert_lsn"), }, @@ -239,7 +239,7 @@ fn generate_internal( // Some records may be not flushed, e.g. non-transactional logical messages. client.execute("select neon_xlogflush(pg_current_wal_insert_lsn())", &[])?; match last_lsn.cmp(&client.pg_current_wal_flush_lsn()?) { - Ordering::Less => bail!("Some records were flushed after the generated WAL"), + Ordering::Less => bail!("Some records were flushed after the crafted WAL"), Ordering::Equal => {} Ordering::Greater => bail!("Reported LSN is greater than flush_lsn"), } @@ -247,7 +247,7 @@ fn generate_internal( } pub fn generate_simple(client: &mut impl postgres::GenericClient) -> Result { - generate_internal(client, |client, _| { + craft_internal(client, |client, _| { client.execute("CREATE table t(x int)", &[])?; Ok(None) }) @@ -256,7 +256,7 @@ pub fn generate_simple(client: &mut impl postgres::GenericClient) -> Result Result { - // Do not use generate_internal because here we end up with flush_lsn exactly on + // Do not use craft_internal because here we end up with flush_lsn exactly on // the segment boundary and insert_lsn after the initial page header, which is unusual. ensure_server_config(client)?; @@ -275,7 +275,7 @@ pub fn generate_last_wal_record_xlog_switch( pub fn generate_last_wal_record_xlog_switch_ends_on_page_boundary( client: &mut impl postgres::GenericClient, ) -> Result { - // Do not use generate_internal because here we end up with flush_lsn exactly on + // Do not use craft_internal because here we end up with flush_lsn exactly on // the segment boundary and insert_lsn after the initial page header, which is unusual. ensure_server_config(client)?; @@ -336,11 +336,11 @@ pub fn generate_last_wal_record_xlog_switch_ends_on_page_boundary( Ok(next_segment) } -fn generate_single_logical_message( +fn craft_single_logical_message( client: &mut impl postgres::GenericClient, transactional: bool, ) -> Result { - generate_internal(client, |client, initial_lsn| { + craft_internal(client, |client, initial_lsn| { ensure!( initial_lsn < PgLsn::from(0x0200_0000 - 1024 * 1024), "Initial LSN is too far in the future" @@ -381,11 +381,11 @@ fn generate_single_logical_message( pub fn generate_wal_record_crossing_segment_followed_by_small_one( client: &mut impl postgres::GenericClient, ) -> Result { - generate_single_logical_message(client, true) + craft_single_logical_message(client, true) } pub fn generate_last_wal_record_crossing_segment( client: &mut C, ) -> Result { - generate_single_logical_message(client, false) + craft_single_logical_message(client, false) } diff --git a/test_runner/batch_others/test_crafted_wal_end.py b/test_runner/batch_others/test_crafted_wal_end.py index 945dfffe4f..d1c46fc73a 100644 --- a/test_runner/batch_others/test_crafted_wal_end.py +++ b/test_runner/batch_others/test_crafted_wal_end.py @@ -1,4 +1,4 @@ -from fixtures.neon_fixtures import NeonEnvBuilder, WalGenerate +from fixtures.neon_fixtures import NeonEnvBuilder, WalCraft from fixtures.log_helper import log import pytest @@ -20,8 +20,8 @@ def test_crafted_wal_end(neon_env_builder: NeonEnvBuilder, wal_type: str): env.neon_cli.create_branch('test_crafted_wal_end') pg = env.postgres.create('test_crafted_wal_end') - gen = WalGenerate(env) - pg.config(gen.postgres_config()) + wal_craft = WalCraft(env) + pg.config(wal_craft.postgres_config()) pg.start() res = pg.safe_psql_many(queries=[ 'CREATE TABLE keys(key int primary key)', @@ -30,7 +30,7 @@ def test_crafted_wal_end(neon_env_builder: NeonEnvBuilder, wal_type: str): ]) assert res[-1][0] == (5050, ) - gen.in_existing(wal_type, pg.connstr()) + wal_craft.in_existing(wal_type, pg.connstr()) log.info("Restarting all safekeepers and pageservers") env.pageserver.stop() diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index d91ea398f9..e2bf7da79d 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1263,13 +1263,13 @@ class NeonCli(AbstractNeonCli): return self.raw_cli(args, check_return_code=check_return_code) -class WalGenerate(AbstractNeonCli): +class WalCraft(AbstractNeonCli): """ - A typed wrapper around the `wal_generate` CLI tool. + A typed wrapper around the `wal_craft` CLI tool. Supports main commands via typed methods and a way to run arbitrary command directly via CLI. """ - COMMAND = 'wal_generate' + COMMAND = 'wal_craft' def postgres_config(self) -> List[str]: res = self.raw_cli(["print-postgres-config"])