rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-06 04:52:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

postgres_ffi/wal_generate: use 'craft' instead of 'generate'

Browse Source 
It does very fine-tuned byte-to-byte WAL crafting, not a sloppy generation.
Hence 'craft' sounds like a better description.
This commit is contained in:
Egor Suvorov
2022-07-08 15:19:53 +03:00
committed by Egor Suvorov
parent 1f5918b36d
commit 60e5dc10e6
8 changed files with 37 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
libs/postgres_ffi/Cargo.toml
View File

@@ -23,7 +23,7 @@ workspace_hack = { version = "0.1", path = "../../workspace_hack" }
[dev-dependencies]
env_logger = "0.9"
postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" }
wal_generate = { path = "wal_generate" }
wal_craft = { path = "wal_craft" }
[build-dependencies]
bindgen = "0.59.1"

10
libs/postgres_ffi/src/xlog_utils.rs
View File

@@ -597,7 +597,7 @@ mod tests {
fn init_logging() {
let _ = env_logger::Builder::from_env(
env_logger::Env::default()
.default_filter_or("wal_generate=info,postgres_ffi::xlog_utils=trace"),
.default_filter_or("wal_craft=info,postgres_ffi::xlog_utils=trace"),
)
.is_test(true)
.try_init();
@@ -609,7 +609,7 @@ mod tests {
expected_end_of_wal_non_partial: Lsn,
last_segment: &str,
) {
use wal_generate::*;
use wal_craft::*;
// 1. Generate some WAL
let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("..")
@@ -683,7 +683,7 @@ mod tests {
init_logging();
test_end_of_wal(
"test_find_end_of_wal_simple",
wal_generate::generate_simple,
wal_craft::generate_simple,
"0/2000000".parse::<Lsn>().unwrap(),
"000000010000000000000001",
);
@@ -694,7 +694,7 @@ mod tests {
init_logging();
test_end_of_wal(
"test_find_end_of_wal_crossing_segment_followed_by_small_one",
wal_generate::generate_wal_record_crossing_segment_followed_by_small_one,
wal_craft::generate_wal_record_crossing_segment_followed_by_small_one,
"0/3000000".parse::<Lsn>().unwrap(),
"000000010000000000000002",
);
@@ -706,7 +706,7 @@ mod tests {
init_logging();
test_end_of_wal(
"test_find_end_of_wal_last_crossing_segment",
wal_generate::generate_last_wal_record_crossing_segment,
wal_craft::generate_last_wal_record_crossing_segment,
"0/3000000".parse::<Lsn>().unwrap(),
"000000010000000000000002",
);

2
libs/postgres_ffi/wal_generate/Cargo.toml → libs/postgres_ffi/wal_craft/Cargo.toml
View File

@@ -1,5 +1,5 @@
[package]
name = "wal_generate"
name = "wal_craft"
version = "0.1.0"
edition = "2021"

24
libs/postgres_ffi/wal_generate/src/bin/wal_generate.rs → libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs
View File

@@ -1,16 +1,14 @@
use anyhow::*;
use clap::{App, Arg, ArgMatches};
use std::str::FromStr;
use wal_generate::*;
use wal_craft::*;
fn main() -> Result<()> {
env_logger::Builder::from_env(
env_logger::Env::default().default_filter_or("wal_generate=info"),
)
.init();
env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("wal_craft=info"))
.init();
let type_arg = &Arg::new("type")
.takes_value(true)
.help("Type of WAL to generate")
.help("Type of WAL to craft")
.possible_values([
"simple",
"last_wal_record_xlog_switch",
@@ -19,15 +17,15 @@ fn main() -> Result<()> {
"wal_record_crossing_segment_followed_by_small_one",
])
.required(true);
let arg_matches = App::new("Postgres WAL generator")
.about("Generates Postgres databases with specific WAL properties")
let arg_matches = App::new("Postgres WAL crafter")
.about("Crafts Postgres databases with specific WAL properties")
.subcommand(
App::new("print-postgres-config")
.about("Print the configuration required for PostgreSQL server before running this script")
)
.subcommand(
App::new("with-initdb")
.about("Generate WAL in a new data directory first initialized with initdb")
.about("Craft WAL in a new data directory first initialized with initdb")
.arg(type_arg)
.arg(
Arg::new("datadir")
@@ -45,7 +43,7 @@ fn main() -> Result<()> {
)
.subcommand(
App::new("in-existing")
.about("Generate WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.")
.about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.")
.arg(type_arg)
.arg(
Arg::new("connection")
@@ -56,7 +54,7 @@ fn main() -> Result<()> {
)
.get_matches();
let wal_generate = |arg_matches: &ArgMatches, client| {
let wal_craft = |arg_matches: &ArgMatches, client| {
let lsn = match arg_matches.value_of("type").unwrap() {
"simple" => generate_simple(client)?,
"last_wal_record_xlog_switch" => generate_last_wal_record_xlog_switch(client)?,
@@ -90,11 +88,11 @@ fn main() -> Result<()> {
};
cfg.initdb()?;
let mut srv = cfg.start_server()?;
wal_generate(arg_matches, &mut srv.connect_with_timeout()?)?;
wal_craft(arg_matches, &mut srv.connect_with_timeout()?)?;
srv.kill();
Ok(())
}
Some(("in-existing", arg_matches)) => wal_generate(
Some(("in-existing", arg_matches)) => wal_craft(
arg_matches,
&mut postgres::Config::from_str(arg_matches.value_of("connection").unwrap())?
.connect(postgres::NoTls)?,

20
libs/postgres_ffi/wal_generate/src/lib.rs → libs/postgres_ffi/wal_craft/src/lib.rs
View File

@@ -218,7 +218,7 @@ pub fn ensure_server_config(client: &mut impl postgres::GenericClient) -> Result
Ok(())
}
fn generate_internal<C: postgres::GenericClient>(
fn craft_internal<C: postgres::GenericClient>(
client: &mut C,
f: impl Fn(&mut C, PgLsn) -> Result<Option<PgLsn>>,
) -> Result<PgLsn> {
@@ -230,7 +230,7 @@ fn generate_internal<C: postgres::GenericClient>(
let last_lsn = match f(client, initial_lsn)? {
None => client.pg_current_wal_insert_lsn()?,
Some(last_lsn) => match last_lsn.cmp(&client.pg_current_wal_insert_lsn()?) {
Ordering::Less => bail!("Some records were inserted after the generated WAL"),
Ordering::Less => bail!("Some records were inserted after the crafted WAL"),
Ordering::Equal => last_lsn,
Ordering::Greater => bail!("Reported LSN is greater than insert_lsn"),
},
@@ -239,7 +239,7 @@ fn generate_internal<C: postgres::GenericClient>(
// Some records may be not flushed, e.g. non-transactional logical messages.
client.execute("select neon_xlogflush(pg_current_wal_insert_lsn())", &[])?;
match last_lsn.cmp(&client.pg_current_wal_flush_lsn()?) {
Ordering::Less => bail!("Some records were flushed after the generated WAL"),
Ordering::Less => bail!("Some records were flushed after the crafted WAL"),
Ordering::Equal => {}
Ordering::Greater => bail!("Reported LSN is greater than flush_lsn"),
}
@@ -247,7 +247,7 @@ fn generate_internal<C: postgres::GenericClient>(
}
pub fn generate_simple(client: &mut impl postgres::GenericClient) -> Result<PgLsn> {
generate_internal(client, |client, _| {
craft_internal(client, |client, _| {
client.execute("CREATE table t(x int)", &[])?;
Ok(None)
})
@@ -256,7 +256,7 @@ pub fn generate_simple(client: &mut impl postgres::GenericClient) -> Result<PgLs
pub fn generate_last_wal_record_xlog_switch(
client: &mut impl postgres::GenericClient,
) -> Result<PgLsn> {
// Do not use generate_internal because here we end up with flush_lsn exactly on
// Do not use craft_internal because here we end up with flush_lsn exactly on
// the segment boundary and insert_lsn after the initial page header, which is unusual.
ensure_server_config(client)?;
@@ -275,7 +275,7 @@ pub fn generate_last_wal_record_xlog_switch(
pub fn generate_last_wal_record_xlog_switch_ends_on_page_boundary(
client: &mut impl postgres::GenericClient,
) -> Result<PgLsn> {
// Do not use generate_internal because here we end up with flush_lsn exactly on
// Do not use craft_internal because here we end up with flush_lsn exactly on
// the segment boundary and insert_lsn after the initial page header, which is unusual.
ensure_server_config(client)?;
@@ -336,11 +336,11 @@ pub fn generate_last_wal_record_xlog_switch_ends_on_page_boundary(
Ok(next_segment)
}
fn generate_single_logical_message(
fn craft_single_logical_message(
client: &mut impl postgres::GenericClient,
transactional: bool,
) -> Result<PgLsn> {
generate_internal(client, |client, initial_lsn| {
craft_internal(client, |client, initial_lsn| {
ensure!(
initial_lsn < PgLsn::from(0x0200_0000 - 1024 * 1024),
"Initial LSN is too far in the future"
@@ -381,11 +381,11 @@ fn generate_single_logical_message(
pub fn generate_wal_record_crossing_segment_followed_by_small_one(
client: &mut impl postgres::GenericClient,
) -> Result<PgLsn> {
generate_single_logical_message(client, true)
craft_single_logical_message(client, true)
}
pub fn generate_last_wal_record_crossing_segment<C: postgres::GenericClient>(
client: &mut C,
) -> Result<PgLsn> {
generate_single_logical_message(client, false)
craft_single_logical_message(client, false)
}