diff --git a/Cargo.lock b/Cargo.lock index 54f4e95c94..080cac04e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4615,6 +4615,29 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "walproposer" +version = "0.1.0" +dependencies = [ + "anyhow", + "bindgen", + "byteorder", + "bytes", + "crc32c", + "env_logger", + "hex", + "log", + "memoffset 0.8.0", + "once_cell", + "postgres", + "rand", + "regex", + "serde", + "thiserror", + "utils", + "workspace_hack", +] + [[package]] name = "want" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index ea22b04124..188f663971 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,6 +142,7 @@ storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main br tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" } tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" } utils = { version = "0.1", path = "./libs/utils/" } +walproposer = { version = "0.1", path = "./libs/walproposer/" } ## Common library dependency workspace_hack = { version = "0.1", path = "./workspace_hack/" } diff --git a/libs/walproposer/Cargo.toml b/libs/walproposer/Cargo.toml new file mode 100644 index 0000000000..778a9f024b --- /dev/null +++ b/libs/walproposer/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "walproposer" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +rand.workspace = true +regex.workspace = true +bytes.workspace = true +byteorder.workspace = true +anyhow.workspace = true +crc32c.workspace = true +hex.workspace = true +once_cell.workspace = true +log.workspace = true +memoffset.workspace = true +thiserror.workspace = true +serde.workspace = true +utils.workspace = true + +workspace_hack.workspace = true + +[dev-dependencies] +env_logger.workspace = true +postgres.workspace = true + +[build-dependencies] +anyhow.workspace = true +bindgen.workspace = true diff --git a/libs/walproposer/bindgen_deps.h b/libs/walproposer/bindgen_deps.h new file mode 100644 index 0000000000..54aa1e6005 --- /dev/null +++ b/libs/walproposer/bindgen_deps.h @@ -0,0 +1,8 @@ +/* + * This header file is the input to bindgen. It includes all the + * PostgreSQL headers that we need to auto-generate Rust structs + * from. If you need to expose a new struct to Rust code, add the + * header here, and whitelist the struct in the build.rs file. + */ +// #include "c.h" +#include "walproposer.h" diff --git a/libs/walproposer/build.rs b/libs/walproposer/build.rs new file mode 100644 index 0000000000..3843add91e --- /dev/null +++ b/libs/walproposer/build.rs @@ -0,0 +1,103 @@ +use std::{path::PathBuf, env, process::Command}; + +use anyhow::{anyhow, Context}; +use bindgen::{callbacks::ParseCallbacks, CargoCallbacks}; + +extern crate bindgen; + +fn main() -> anyhow::Result<()> { + // Tell cargo to invalidate the built crate whenever the wrapper changes + println!("cargo:rerun-if-changed=bindgen_deps.h,walproposer.c,walproposer.h"); + println!("cargo:rustc-link-lib=dylib=walproposer"); + println!("cargo:rustc-link-search=/Users/arthur/zen/zenith/libs/walproposer"); + + if !std::process::Command::new("./build.sh") + .output() + .expect("could not spawn `clang`") + .status + .success() + { + // Panic if the command was not successful. + panic!("could not compile object file"); + } + + // println!("cargo:rustc-link-lib=dylib=neon"); + // println!("cargo:rustc-link-search=/Users/arthur/zen/zenith/pg_install/build/neon-v15"); + // println!("cargo:rustc-link-arg=-Wl,-rpath,/Users/arthur/zen/zenith/pg_install/build/neon-v15"); + + // // Finding the location of C headers for the Postgres server: + // // - if POSTGRES_INSTALL_DIR is set look into it, otherwise look into `/pg_install` + // // - if there's a `bin/pg_config` file use it for getting include server, otherwise use `/pg_install/{PG_MAJORVERSION}/include/postgresql/server` + // let pg_install_dir = if let Some(postgres_install_dir) = env::var_os("POSTGRES_INSTALL_DIR") { + // postgres_install_dir.into() + // } else { + // PathBuf::from("pg_install") + // }; + + // let pg_version = "v15"; + // let mut pg_install_dir_versioned = pg_install_dir.join(pg_version); + // if pg_install_dir_versioned.is_relative() { + // let cwd = env::current_dir().context("Failed to get current_dir")?; + // pg_install_dir_versioned = cwd.join("..").join("..").join(pg_install_dir_versioned); + // } + + // let pg_config_bin = pg_install_dir_versioned + // .join(pg_version) + // .join("bin") + // .join("pg_config"); + // let inc_server_path: String = if pg_config_bin.exists() { + // let output = Command::new(pg_config_bin) + // .arg("--includedir-server") + // .output() + // .context("failed to execute `pg_config --includedir-server`")?; + + // if !output.status.success() { + // panic!("`pg_config --includedir-server` failed") + // } + + // String::from_utf8(output.stdout) + // .context("pg_config output is not UTF-8")? + // .trim_end() + // .into() + // } else { + // let server_path = pg_install_dir_versioned + // .join("include") + // .join("postgresql") + // .join("server") + // .into_os_string(); + // server_path + // .into_string() + // .map_err(|s| anyhow!("Bad postgres server path {s:?}"))? + // }; + + // let inc_pgxn_path = "/Users/arthur/zen/zenith/pgxn/neon"; + + // TODO: build a libwalproposer.a + + // The bindgen::Builder is the main entry point + // to bindgen, and lets you build up options for + // the resulting bindings. + let bindings = bindgen::Builder::default() + // The input header we would like to generate + // bindings for. + .header("bindgen_deps.h") + // Tell cargo to invalidate the built crate whenever any of the + // included header files changed. + .parse_callbacks(Box::new(CargoCallbacks)) + .allowlist_function("WalProposerRust") + // .clang_arg(format!("-I{inc_server_path}")) + // .clang_arg(format!("-I{inc_pgxn_path}")) + // Finish the builder and generate the bindings. + .generate() + // Unwrap the Result and panic on failure. + .expect("Unable to generate bindings"); + + // Write the bindings to the $OUT_DIR/bindings.rs file. + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("bindings.rs"); + bindings + .write_to_file(out_path) + .expect("Couldn't write bindings!"); + + + Ok(()) +} diff --git a/libs/walproposer/build.sh b/libs/walproposer/build.sh new file mode 100755 index 0000000000..6af45037fb --- /dev/null +++ b/libs/walproposer/build.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# cc -c -o walproposer.o \ +# -static \ +# -I /Users/arthur/zen/zenith/pg_install/v15/include/postgresql/server \ +# /Users/arthur/zen/zenith/pgxn/neon/walproposer.c + +# cc -shared \ +# -I /Users/arthur/zen/zenith/pg_install/v15/include/postgresql/server \ +# -I /Users/arthur/zen/zenith/pgxn/neon \ +# -L /Users/arthur/zen/zenith/pg_install/build/v15/src/common \ +# -L /Users/arthur/zen/zenith/pg_install/build/v15/src/port \ +# -lpgcommon -lpgport -lz -lreadline -lm \ +# -o walproposer.so walproposer.o + +clang -c -o walproposer.o walproposer.c +ar rcs libwalproposer.a walproposer.o \ No newline at end of file diff --git a/libs/walproposer/libwalproposer.a b/libs/walproposer/libwalproposer.a new file mode 100644 index 0000000000..bd8b639139 Binary files /dev/null and b/libs/walproposer/libwalproposer.a differ diff --git a/libs/walproposer/src/lib.rs b/libs/walproposer/src/lib.rs new file mode 100644 index 0000000000..939b1c36b7 --- /dev/null +++ b/libs/walproposer/src/lib.rs @@ -0,0 +1,12 @@ +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +pub mod bindings { + include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +} + +pub use bindings::WalProposerRust; + +#[cfg(test)] +mod test; diff --git a/libs/walproposer/src/test.rs b/libs/walproposer/src/test.rs new file mode 100644 index 0000000000..b6a907d95d --- /dev/null +++ b/libs/walproposer/src/test.rs @@ -0,0 +1,6 @@ +use crate::WalProposerRust; + +#[test] +fn run_test() { + unsafe { WalProposerRust(); } +} \ No newline at end of file diff --git a/libs/walproposer/walproposer.c b/libs/walproposer/walproposer.c new file mode 100644 index 0000000000..07af225638 --- /dev/null +++ b/libs/walproposer/walproposer.c @@ -0,0 +1,6 @@ +#include + +int WalProposerRust(void) { + puts("This is a shared library test...\n"); + return 42; +} diff --git a/libs/walproposer/walproposer.h b/libs/walproposer/walproposer.h new file mode 100644 index 0000000000..d5e3632df8 --- /dev/null +++ b/libs/walproposer/walproposer.h @@ -0,0 +1 @@ +int WalProposerRust(void); \ No newline at end of file diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index ec377dbb1e..61a96e3925 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -14,12 +14,14 @@ OBJS = \ walproposer_utils.o PG_CPPFLAGS = -I$(libpq_srcdir) +PG_LIBS = $(libpq) +PG_LIBS_INTERNAL = $(libpq) SHLIB_LINK_INTERNAL = $(libpq) - EXTENSION = neon DATA = neon--1.0.sql PGFILEDESC = "neon - cloud storage for PostgreSQL" +# PROGRAM = boop PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/safekeeper/src/simtest/mod.rs b/safekeeper/src/simtest/mod.rs index 57ab33a034..6ec9f9157e 100644 --- a/safekeeper/src/simtest/mod.rs +++ b/safekeeper/src/simtest/mod.rs @@ -17,12 +17,12 @@ use crate::{ fn run_test() { let delay = Delay { min: 1, - max: 10, + max: 60, fail_prob: 0.4, }; let network = NetworkOptions { - timeout: Some(1000), + timeout: Some(50), connect_delay: delay.clone(), send_delay: delay.clone(), };