diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 24c4e776df..2296807d2d 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -99,7 +99,9 @@ jobs: # toes, ensure that the toolchain is up-to-date beforehand. - name: Update rust toolchain run: | - rustup update + rustup --version && + rustup update && + rustup show - name: Cache cargo deps uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 diff --git a/Cargo.lock b/Cargo.lock index 4c9cfa97e1..0d4dc10149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1083,6 +1083,25 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cbindgen" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "975982cdb7ad6a142be15bdf84aea7ec6a9e5d4d797c004d43185b24cfe4e684" +dependencies = [ + "clap", + "heck", + "indexmap 2.9.0", + "log", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn 2.0.100", + "tempfile", + "toml", +] + [[package]] name = "cc" version = "1.2.16" @@ -1267,6 +1286,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "communicator" +version = "0.1.0" +dependencies = [ + "cbindgen", + "neon-shmem", + "workspace_hack", +] + [[package]] name = "compute_api" version = "0.1.0" @@ -8693,6 +8721,7 @@ dependencies = [ "num-iter", "num-rational", "num-traits", + "once_cell", "p256 0.13.2", "parquet", "prettyplease", diff --git a/Cargo.toml b/Cargo.toml index 7728f6d8fe..68016a08a9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ members = [ "libs/proxy/postgres-types2", "libs/proxy/tokio-postgres2", "endpoint_storage", + "pgxn/neon/communicator", ] [workspace.package] @@ -255,6 +256,7 @@ desim = { version = "0.1", path = "./libs/desim" } endpoint_storage = { version = "0.0.1", path = "./endpoint_storage/" } http-utils = { version = "0.1", path = "./libs/http-utils/" } metrics = { version = "0.1", path = "./libs/metrics/" } +neon-shmem = { version = "0.1", path = "./libs/neon-shmem/" } pageserver = { path = "./pageserver" } pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" } pageserver_client = { path = "./pageserver/client" } @@ -284,6 +286,7 @@ walproposer = { version = "0.1", path = "./libs/walproposer/" } workspace_hack = { version = "0.1", path = "./workspace_hack/" } ## Build dependencies +cbindgen = "0.29.0" criterion = "0.5.1" rcgen = "0.13" rstest = "0.18" diff --git a/Dockerfile b/Dockerfile index d518370ab8..55b87d4012 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,18 @@ ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR} ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA} ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA} -# Build Postgres +# Naive way: +# +# 1. COPY . . +# 1. make neon-pg-ext +# 2. cargo build +# +# But to enable docker to cache intermediate layers, we perform a few preparatory steps: +# +# - Build all postgres versions, depending on just the contents of vendor/ +# - Use cargo chef to build all rust dependencies + +# 1. Build all postgres versions FROM $REPOSITORY/$IMAGE:$TAG AS pg-build WORKDIR /home/nonroot @@ -38,17 +49,15 @@ COPY --chown=nonroot vendor/postgres-v14 vendor/postgres-v14 COPY --chown=nonroot vendor/postgres-v15 vendor/postgres-v15 COPY --chown=nonroot vendor/postgres-v16 vendor/postgres-v16 COPY --chown=nonroot vendor/postgres-v17 vendor/postgres-v17 -COPY --chown=nonroot pgxn pgxn COPY --chown=nonroot Makefile Makefile COPY --chown=nonroot postgres.mk postgres.mk COPY --chown=nonroot scripts/ninstall.sh scripts/ninstall.sh ENV BUILD_TYPE=release RUN set -e \ - && mold -run make -j $(nproc) -s neon-pg-ext \ - && tar -C pg_install -czf /home/nonroot/postgres_install.tar.gz . + && mold -run make -j $(nproc) -s postgres -# Prepare cargo-chef recipe +# 2. Prepare cargo-chef recipe FROM $REPOSITORY/$IMAGE:$TAG AS plan WORKDIR /home/nonroot @@ -56,23 +65,22 @@ COPY --chown=nonroot . . RUN cargo chef prepare --recipe-path recipe.json -# Build neon binaries +# Main build image FROM $REPOSITORY/$IMAGE:$TAG AS build WORKDIR /home/nonroot ARG GIT_VERSION=local ARG BUILD_TAG - -COPY --from=pg-build /home/nonroot/pg_install/v14/include/postgresql/server pg_install/v14/include/postgresql/server -COPY --from=pg-build /home/nonroot/pg_install/v15/include/postgresql/server pg_install/v15/include/postgresql/server -COPY --from=pg-build /home/nonroot/pg_install/v16/include/postgresql/server pg_install/v16/include/postgresql/server -COPY --from=pg-build /home/nonroot/pg_install/v17/include/postgresql/server pg_install/v17/include/postgresql/server -COPY --from=plan /home/nonroot/recipe.json recipe.json - ARG ADDITIONAL_RUSTFLAGS="" +# 3. Build cargo dependencies. Note that this step doesn't depend on anything else than +# `recipe.json`, so the layer can be reused as long as none of the dependencies change. +COPY --from=plan /home/nonroot/recipe.json recipe.json RUN set -e \ && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo chef cook --locked --release --recipe-path recipe.json +# Perform the main build. We reuse the Postgres build artifacts from the intermediate 'pg-build' +# layer, and the cargo dependencies built in the previous step. +COPY --chown=nonroot --from=pg-build /home/nonroot/pg_install/ pg_install COPY --chown=nonroot . . RUN set -e \ @@ -87,10 +95,10 @@ RUN set -e \ --bin endpoint_storage \ --bin neon_local \ --bin storage_scrubber \ - --locked --release + --locked --release \ + && mold -run make -j $(nproc) -s neon-pg-ext -# Build final image -# +# Assemble the final image FROM $BASE_IMAGE_SHA WORKDIR /data @@ -130,12 +138,15 @@ COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy COPY --from=build --chown=neon:neon /home/nonroot/target/release/endpoint_storage /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/neon_local /usr/local/bin COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_scrubber /usr/local/bin +COPY --from=build /home/nonroot/pg_install/v14 /usr/local/v14/ +COPY --from=build /home/nonroot/pg_install/v15 /usr/local/v15/ +COPY --from=build /home/nonroot/pg_install/v16 /usr/local/v16/ +COPY --from=build /home/nonroot/pg_install/v17 /usr/local/v17/ -COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/ -COPY --from=pg-build /home/nonroot/pg_install/v15 /usr/local/v15/ -COPY --from=pg-build /home/nonroot/pg_install/v16 /usr/local/v16/ -COPY --from=pg-build /home/nonroot/pg_install/v17 /usr/local/v17/ -COPY --from=pg-build /home/nonroot/postgres_install.tar.gz /data/ +# Deprecated: Old deployment scripts use this tarball which contains all the Postgres binaries. +# That's obsolete, since all the same files are also present under /usr/local/v*. But to keep the +# old scripts working for now, create the tarball. +RUN tar -C /usr/local -cvzf /data/postgres_install.tar.gz v14 v15 v16 v17 # By default, pageserver uses `.neon/` working directory in WORKDIR, so create one and fill it with the dummy config. # Now, when `docker run ... pageserver` is run, it can start without errors, yet will have some default dummy values. diff --git a/Makefile b/Makefile index 8ebd27f7c5..4b31e26810 100644 --- a/Makefile +++ b/Makefile @@ -30,11 +30,18 @@ ifeq ($(BUILD_TYPE),release) PG_CFLAGS += -O2 -g3 $(CFLAGS) PG_LDFLAGS = $(LDFLAGS) CARGO_PROFILE ?= --profile=release + # NEON_CARGO_ARTIFACT_TARGET_DIR is the directory where `cargo build` places + # the final build artifacts. There is unfortunately no easy way of changing + # it to a fully predictable path, nor to extract the path with a simple + # command. See https://github.com/rust-lang/cargo/issues/9661 and + # https://github.com/rust-lang/cargo/issues/6790. + NEON_CARGO_ARTIFACT_TARGET_DIR = $(ROOT_PROJECT_DIR)/target/release else ifeq ($(BUILD_TYPE),debug) PG_CONFIGURE_OPTS = --enable-debug --with-openssl --enable-cassert --enable-depend PG_CFLAGS += -O0 -g3 $(CFLAGS) PG_LDFLAGS = $(LDFLAGS) CARGO_PROFILE ?= --profile=dev + NEON_CARGO_ARTIFACT_TARGET_DIR = $(ROOT_PROJECT_DIR)/target/debug else $(error Bad build type '$(BUILD_TYPE)', see Makefile for options) endif @@ -115,10 +122,13 @@ cargo-target-dir: test -e target/CACHEDIR.TAG || echo "$(CACHEDIR_TAG_CONTENTS)" > target/CACHEDIR.TAG .PHONY: neon-pg-ext-% -neon-pg-ext-%: postgres-install-% +neon-pg-ext-%: postgres-install-% cargo-target-dir +@echo "Compiling neon-specific Postgres extensions for $*" mkdir -p $(BUILD_DIR)/pgxn-$* - $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config COPT='$(COPT)' \ + $(MAKE) PG_CONFIG="$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config" COPT='$(COPT)' \ + NEON_CARGO_ARTIFACT_TARGET_DIR="$(NEON_CARGO_ARTIFACT_TARGET_DIR)" \ + CARGO_BUILD_FLAGS="$(CARGO_BUILD_FLAGS)" \ + CARGO_PROFILE="$(CARGO_PROFILE)" \ -C $(BUILD_DIR)/pgxn-$*\ -f $(ROOT_PROJECT_DIR)/pgxn/Makefile install diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 9f4e3e7d5e..0dd32011fb 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1636,11 +1636,14 @@ RUN make install USE_PGXS=1 -j $(getconf _NPROCESSORS_ONLN) # compile neon extensions # ######################################################################################### -FROM pg-build AS neon-ext-build +FROM pg-build-with-cargo AS neon-ext-build ARG PG_VERSION -COPY pgxn/ pgxn/ -RUN make -j $(getconf _NPROCESSORS_ONLN) -C pgxn -s install-compute +USER root +COPY . . + +RUN make -j $(getconf _NPROCESSORS_ONLN) -C pgxn -s install-compute \ + BUILD_TYPE=release CARGO_BUILD_FLAGS="--locked --release" NEON_CARGO_ARTIFACT_TARGET_DIR="$(pwd)/target/release" ######################################################################################### # diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index 9bce0e798a..bf7aeb4108 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -22,7 +22,8 @@ OBJS = \ walproposer.o \ walproposer_pg.o \ neon_ddl_handler.o \ - walsender_hooks.o + walsender_hooks.o \ + $(NEON_CARGO_ARTIFACT_TARGET_DIR)/libcommunicator.a PG_CPPFLAGS = -I$(libpq_srcdir) SHLIB_LINK_INTERNAL = $(libpq) @@ -54,6 +55,17 @@ WALPROP_OBJS = \ neon_utils.o \ walproposer_compat.o +# libcommunicator.a is built by cargo from the Rust sources under communicator/ +# subdirectory. `cargo build` also generates communicator_bindings.h. +neon.o: communicator/communicator_bindings.h + +$(NEON_CARGO_ARTIFACT_TARGET_DIR)/libcommunicator.a communicator/communicator_bindings.h &: + (cd $(srcdir)/communicator && cargo build $(CARGO_BUILD_FLAGS) $(CARGO_PROFILE)) + +# Force `cargo build` every time. Some of the Rust sources might have +# changed. +.PHONY: $(NEON_CARGO_ARTIFACT_TARGET_DIR)/libcommunicator.a communicator/communicator_bindings.h + .PHONY: walproposer-lib walproposer-lib: CPPFLAGS += -DWALPROPOSER_LIB walproposer-lib: libwalproposer.a; diff --git a/pgxn/neon/communicator/.gitignore b/pgxn/neon/communicator/.gitignore new file mode 100644 index 0000000000..d713be0a35 --- /dev/null +++ b/pgxn/neon/communicator/.gitignore @@ -0,0 +1,2 @@ +# generated file (with cbindgen, see build.rs) +communicator_bindings.h diff --git a/pgxn/neon/communicator/Cargo.toml b/pgxn/neon/communicator/Cargo.toml new file mode 100644 index 0000000000..e95a269d90 --- /dev/null +++ b/pgxn/neon/communicator/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "communicator" +version = "0.1.0" +license.workspace = true +edition.workspace = true + +[lib] +crate-type = ["staticlib"] + +[features] +# 'testing' feature is currently unused in the communicator, but we accept it for convenience of +# calling build scripts, so that you can pass the same feature to all packages. +testing = [] + +[dependencies] +neon-shmem.workspace = true +workspace_hack = { version = "0.1", path = "../../../workspace_hack" } + +[build-dependencies] +cbindgen.workspace = true diff --git a/pgxn/neon/communicator/README.md b/pgxn/neon/communicator/README.md new file mode 100644 index 0000000000..8169ae72b5 --- /dev/null +++ b/pgxn/neon/communicator/README.md @@ -0,0 +1,8 @@ +This package will evolve into a "compute-pageserver communicator" +process and machinery. For now, it's just a dummy that doesn't do +anything interesting, but it allows us to test the compilation and +linking of Rust code into the Postgres extensions. + +At compilation time, pgxn/neon/communicator/ produces a static +library, libcommunicator.a. It is linked to the neon.so extension +library. diff --git a/pgxn/neon/communicator/build.rs b/pgxn/neon/communicator/build.rs new file mode 100644 index 0000000000..2b83b4238d --- /dev/null +++ b/pgxn/neon/communicator/build.rs @@ -0,0 +1,20 @@ +use std::env; + +fn main() -> Result<(), Box> { + let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + + match cbindgen::generate(crate_dir) { + Ok(bindings) => { + bindings.write_to_file("communicator_bindings.h"); + } + Err(cbindgen::Error::ParseSyntaxError { .. }) => { + // This means there was a syntax error in the Rust sources. Don't panic, because + // we want the build to continue and the Rust compiler to hit the error. The + // Rust compiler produces a better error message than cbindgen. + eprintln!("Generating C bindings failed because of a Rust syntax error"); + } + Err(err) => panic!("Unable to generate C bindings: {err:?}"), + }; + + Ok(()) +} diff --git a/pgxn/neon/communicator/cbindgen.toml b/pgxn/neon/communicator/cbindgen.toml new file mode 100644 index 0000000000..72e0c8174a --- /dev/null +++ b/pgxn/neon/communicator/cbindgen.toml @@ -0,0 +1,4 @@ +language = "C" + +[enum] +prefix_with_name = true diff --git a/pgxn/neon/communicator/src/lib.rs b/pgxn/neon/communicator/src/lib.rs new file mode 100644 index 0000000000..24c180d37d --- /dev/null +++ b/pgxn/neon/communicator/src/lib.rs @@ -0,0 +1,6 @@ +/// dummy function, just to test linking Rust functions into the C +/// extension +#[unsafe(no_mangle)] +pub extern "C" fn communicator_dummy(arg: u32) -> u32 { + arg + 1 +} diff --git a/pgxn/neon/neon.c b/pgxn/neon/neon.c index 3b2a4d3f2f..9e0ca16fed 100644 --- a/pgxn/neon/neon.c +++ b/pgxn/neon/neon.c @@ -43,6 +43,9 @@ #include "storage/ipc.h" #endif +/* the rust bindings, generated by cbindgen */ +#include "communicator/communicator_bindings.h" + PG_MODULE_MAGIC; void _PG_init(void); @@ -452,6 +455,9 @@ _PG_init(void) shmem_startup_hook = neon_shmem_startup_hook; #endif + /* dummy call to a Rust function in the communicator library, to check that it works */ + (void) communicator_dummy(123); + pg_init_libpagestore(); lfc_init(); pg_init_walproposer(); diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index e9a77ca2d6..fb10e27d2a 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -68,6 +68,7 @@ num-integer = { version = "0.1", features = ["i128"] } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] } num-traits = { version = "0.2", features = ["i128", "libm"] } +once_cell = { version = "1" } p256 = { version = "0.13", features = ["jwk"] } parquet = { version = "53", default-features = false, features = ["zstd"] } prost = { version = "0.13", features = ["no-recursion-limit", "prost-derive"] } @@ -112,10 +113,13 @@ zstd-sys = { version = "2", default-features = false, features = ["legacy", "std [build-dependencies] ahash = { version = "0.8" } +anstream = { version = "0.6" } anyhow = { version = "1", features = ["backtrace"] } bytes = { version = "1", features = ["serde"] } cc = { version = "1", default-features = false, features = ["parallel"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } +clap = { version = "4", features = ["derive", "env", "string"] } +clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] } either = { version = "1" } getrandom = { version = "0.2", default-features = false, features = ["std"] } half = { version = "2", default-features = false, features = ["num-traits"] } @@ -133,6 +137,7 @@ num-integer = { version = "0.1", features = ["i128"] } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } num-rational = { version = "0.4", default-features = false, features = ["num-bigint-std", "std"] } num-traits = { version = "0.2", features = ["i128", "libm"] } +once_cell = { version = "1" } parquet = { version = "53", default-features = false, features = ["zstd"] } prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] } proc-macro2 = { version = "1" } @@ -142,6 +147,7 @@ regex = { version = "1" } regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] } regex-syntax = { version = "0.8" } serde = { version = "1", features = ["alloc", "derive"] } +serde_json = { version = "1", features = ["alloc", "raw_value"] } syn = { version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } toml_edit = { version = "0.22", features = ["serde"] }