From 002cd8ed5be69e7c65c398a33fa0f4be869279d2 Mon Sep 17 00:00:00 2001 From: Stas Kelvich Date: Mon, 31 May 2021 12:47:49 +0300 Subject: [PATCH] Dockerfile for pageserver. --- .dockerignore | 13 +++++++ Dockerfile | 80 +++++++++++++++++++++++++++++++++++++++++++ Makefile | 2 +- pageserver/Cargo.toml | 3 +- 4 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..ccad1e364c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +**/.git/ +**/__pycache__ +**/.pytest_cache + +/target +/tmp_check +/tmp_install +/tmp_check_cli +/test_output +/.vscode +/.zenith +/integration_tests/.zenith +/Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..e3c58c8bef --- /dev/null +++ b/Dockerfile @@ -0,0 +1,80 @@ +# +# Docker image for console integration testing. +# +# We may also reuse it in CI to unify installation process and as a general binaries building +# tool for production servers. +# +# Dynamic linking is used for librocksdb and libstdc++ bacause librocksdb-sys calls +# bindgen with 'dynamic' feature flag. This also prevents usage of dockerhub alpine-rust +# images which are statically linked and have guards against any dlopen. I would rather +# prefer all static binaries so we may change the way librocksdb-sys builds or wait until +# we will have our own storage and drop rockdb dependency. +# + +# +# build postgres separately -- this layer will be rebuilt only if one of +# mentioned paths will get any changes +# +FROM alpine:3.13 as pg-build +RUN apk add --update clang llvm compiler-rt compiler-rt-static lld musl-dev binutils \ + make bison flex readline-dev zlib-dev perl linux-headers +WORKDIR zenith +COPY ./vendor/postgres vendor/postgres +COPY ./Makefile Makefile +# Build using clang and lld +RUN CC='clang' LD='lld' CFLAGS='-fuse-ld=lld --rtlib=compiler-rt' make postgres -j4 + +# +# Calculate cargo dependencies. +# This will always run, but only generate recipe.json with list of dependencies without +# installing them. +# +FROM alpine:20210212 as cargo-deps-inspect +RUN apk add --update rust cargo +RUN cargo install cargo-chef +WORKDIR zenith +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +# +# Build cargo dependencies. +# This temp cantainner would be build only if recipe.json was changed. +# +FROM alpine:20210212 as deps-build +RUN apk add --update rust cargo openssl-dev clang build-base +# rust-rocksdb can be built against system-wide rocksdb -- that saves about +# 10 minutes during build. Rocksdb apk package is in testing now, but use it +# anyway. In case of any troubles we can download and build rocksdb here manually +# (to cache it as a docker layer). +RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev +WORKDIR zenith +COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server +COPY --from=cargo-deps-inspect /root/.cargo/bin/cargo-chef /root/.cargo/bin/ +COPY --from=cargo-deps-inspect /zenith/recipe.json recipe.json +RUN ROCKSDB_LIB_DIR=/usr/lib/ cargo chef cook --release --recipe-path recipe.json + +# +# Build zenith binaries +# +FROM alpine:20210212 as build +RUN apk add --update rust cargo openssl-dev clang build-base +RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb-dev +WORKDIR zenith +COPY . . +# Copy cached dependencies +COPY --from=pg-build /zenith/tmp_install/include/postgresql/server tmp_install/include/postgresql/server +COPY --from=deps-build /zenith/target target +COPY --from=deps-build /root/.cargo /root/.cargo +RUN cargo build --release + +# +# Copy binaries to resulting image. +# build-base hare to provide libstdc++ (it will also bring gcc, but leave it this way until we figure +# out how to statically link rocksdb or avoid at all). +# +FROM alpine:3.13 +RUN apk add --update openssl build-base +RUN apk --no-cache --update --repository https://dl-cdn.alpinelinux.org/alpine/edge/testing add rocksdb +WORKDIR zenith +COPY --from=build /zenith/target/release/pageserver /usr/local/bin +COPY --from=pg-build /zenith/tmp_install /usr/local diff --git a/Makefile b/Makefile index 1c0bd8bee8..c5f152042b 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ tmp_install/build/config.status: +@echo "Configuring postgres build" mkdir -p tmp_install/build (cd tmp_install/build && \ - ../../vendor/postgres/configure CFLAGS='-O0' --enable-debug --enable-cassert \ + ../../vendor/postgres/configure CFLAGS='-O0 $(CFLAGS)' --enable-debug --enable-cassert \ --enable-depend --prefix=$(abspath tmp_install) > configure.log) # nicer alias for running 'configure' diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index cf6f720b79..d513f9e446 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -30,7 +30,8 @@ tokio-stream = { version = "0.1.4" } postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" } postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" } postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="9eb0dbfbeb6a6c1b79099b9f7ae4a8c021877858" } -rocksdb = "0.16.0" +# by default rust-rocksdb tries to build a lot of compression algos. Use lz4 only for now as it is simplest dependency. +rocksdb = { version = "0.16.0", features = ["lz4"], default-features = false } anyhow = "1.0" crc32c = "0.6.0" walkdir = "2"