From 3a07c2d4885e107bd61bfd843765f0832f61a3ac Mon Sep 17 00:00:00 2001 From: fcdm <128653800+fcdm@users.noreply.github.com> Date: Mon, 19 Feb 2024 11:17:13 +0000 Subject: [PATCH] wip: merged compute image --- .github/workflows/build_and_test.yml | 77 +++++++++++++ Dockerfile.compute-node-merged | 83 ++++++++++++++ Dockerfile.compute-node-simple | 159 +++++++++++++++++++++++++++ 3 files changed, 319 insertions(+) create mode 100644 Dockerfile.compute-node-merged create mode 100644 Dockerfile.compute-node-simple diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 2a1c79e437..75f45736f2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1204,3 +1204,80 @@ jobs: time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/${PREFIX}/${FILENAME} done + + compute-node-image-merged-base: + needs: [ check-permissions, build-buildtools-image, tag ] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + version: [ v14, v15, v16 ] + defaults: + run: + shell: sh -eu {0} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Configure Docker Hub login + run: | + DOCKERHUB_AUTH=$(echo -n "${{ secrets.NEON_DOCKERHUB_USERNAME }}:${{ secrets.NEON_DOCKERHUB_PASSWORD }}" | base64) + echo "::add-mask::${DOCKERHUB_AUTH}" + + cat <<-EOF > ~/.docker/config.json + { + "auths": { + "https://index.docker.io/v1/": { + "auth": "${DOCKERHUB_AUTH}" + } + } + } + EOF + + - name: Build merged image base + run: | + docker image build . -f Dockerfile.compute-node-simple -t neondatabase/tmp-compute-node-merged-base-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ + --build-arg PG_VERSION=${{ matrix.version }} \ + --build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}} \ + --build-arg TAG=${{needs.build-buildtools-image.outputs.build-tools-tag}} + docker image push neondatabase/tmp-compute-node-merged-base-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} + + compute-node-image-merged: + needs: [ tag, compute-node-image-merged-base ] + runs-on: ubuntu-latest + defaults: + run: + shell: sh -eu {0} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Configure Docker Hub login + run: | + DOCKERHUB_AUTH=$(echo -n "${{ secrets.NEON_DOCKERHUB_USERNAME }}:${{ secrets.NEON_DOCKERHUB_PASSWORD }}" | base64) + echo "::add-mask::${DOCKERHUB_AUTH}" + + echo ~ + cat <<-EOF > ~/.docker/config.json + { + "auths": { + "https://index.docker.io/v1/": { + "auth": "${DOCKERHUB_AUTH}" + } + } + } + EOF + + - name: Build merged image + run: | + docker image build . -f Dockerfile.compute-node-merged -t neondatabase/tmp-compute-node-merged:${{needs.tag.outputs.build-tag}} \ + --build-arg TAG=${{needs.tag.outputs.build-tag}} + docker image push neondatabase/tmp-compute-node-merged:${{needs.tag.outputs.build-tag}} diff --git a/Dockerfile.compute-node-merged b/Dockerfile.compute-node-merged new file mode 100644 index 0000000000..1f9c8396c1 --- /dev/null +++ b/Dockerfile.compute-node-merged @@ -0,0 +1,83 @@ +ARG TAG +FROM neondatabase/tmp-compute-node-merged-base-v14:$TAG as pg14 +FROM neondatabase/tmp-compute-node-merged-base-v15:$TAG as pg15 +FROM neondatabase/tmp-compute-node-merged-base-v16:$TAG as pg16 + +######################################################################################### +# +# Compile and run the Neon-specific `compute_ctl` binary +# +######################################################################################### +FROM neondatabase/build-tools:pinned AS compute-tools +ARG BUILD_TAG +ENV BUILD_TAG=$BUILD_TAG + +USER nonroot +# Copy entire project to get Cargo.* files with proper dependencies for the whole project +COPY --chown=nonroot . . +RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto + +######################################################################################### +# +# Final layer +# Put it all together into the final image +# +######################################################################################### +FROM debian:bullseye-slim +ARG TAG +# Add user postgres +RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \ + echo "postgres:test_console_pass" | chpasswd && \ + mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \ + mkdir /var/db/postgres/pgbouncer && \ + chown -R postgres:postgres /var/db/postgres && \ + chmod 0750 /var/db/postgres/compute && \ + chmod 0750 /var/db/postgres/pgbouncer && \ + echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig && \ + # create folder for file cache + mkdir -p -m 777 /neon/cache + +COPY --from=pg14 --chown=postgres /usr/local/pgsql /usr/local/pgsql-v14 +COPY --from=pg15 --chown=postgres /usr/local/pgsql /usr/local/pgsql-v15 +COPY --from=pg16 --chown=postgres /usr/local/pgsql /usr/local/pgsql-v16 +COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl + +# Install: +# libreadline8 for psql +# libicu67, locales for collations (including ICU and plpgsql_check) +# liblz4-1 for lz4 +# libossp-uuid16 for extension ossp-uuid +# libgeos, libgdal, libsfcgal1, libproj and libprotobuf-c1 for PostGIS +# libxml2, libxslt1.1 for xml2 +# libzstd1 for zstd +# libboost* for rdkit +# ca-certificates for communicating with s3 by compute_ctl +RUN apt update && \ + apt install --no-install-recommends -y \ + gdb \ + libicu67 \ + liblz4-1 \ + libreadline8 \ + libboost-iostreams1.74.0 \ + libboost-regex1.74.0 \ + libboost-serialization1.74.0 \ + libboost-system1.74.0 \ + libossp-uuid16 \ + libgeos-c1v5 \ + libgdal28 \ + libproj19 \ + libprotobuf-c1 \ + libsfcgal1 \ + libxml2 \ + libxslt1.1 \ + libzstd1 \ + libcurl4-openssl-dev \ + locales \ + procps \ + ca-certificates && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 + +ENV LANG en_US.utf8 +USER postgres +ENTRYPOINT ["/usr/local/bin/compute_ctl"] diff --git a/Dockerfile.compute-node-simple b/Dockerfile.compute-node-simple new file mode 100644 index 0000000000..e39bb8cea1 --- /dev/null +++ b/Dockerfile.compute-node-simple @@ -0,0 +1,159 @@ +ARG PG_VERSION +ARG REPOSITORY=neondatabase +ARG IMAGE=build-tools +ARG TAG=pinned +ARG BUILD_TAG + +######################################################################################### +# +# Layer "build-deps" +# +######################################################################################### +FROM debian:bullseye-slim AS build-deps +RUN apt update && \ + apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ + zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev \ + libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd + +######################################################################################### +# +# Layer "pg-build" +# Build Postgres from the neon postgres repository. +# +######################################################################################### +FROM build-deps AS pg-build +ARG PG_VERSION +COPY vendor/postgres-${PG_VERSION} postgres +RUN cd postgres && \ + export CONFIGURE_CMD="./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp \ + --with-icu --with-libxml --with-libxslt --with-lz4" && \ + if [ "${PG_VERSION}" != "v14" ]; then \ + # zstd is available only from PG15 + export CONFIGURE_CMD="${CONFIGURE_CMD} --with-zstd"; \ + fi && \ + eval $CONFIGURE_CMD && \ + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \ + # Install headers + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \ + make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install && \ + # Enable some of contrib extensions + echo 'trusted = true' >> /usr/local/pgsql/share/extension/autoinc.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/bloom.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/insert_username.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/intagg.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/moddatetime.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_stat_statements.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrowlocks.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgstattuple.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/refint.control && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/xml2.control && \ + # We need to grant EXECUTE on pg_stat_statements_reset() to neon_superuser. + # In vanilla postgres this function is limited to Postgres role superuser. + # In neon we have neon_superuser role that is not a superuser but replaces superuser in some cases. + # We could add the additional grant statements to the postgres repository but it would be hard to maintain, + # whenever we need to pick up a new postgres version and we want to limit the changes in our postgres fork, + # so we do it here. + old_list="pg_stat_statements--1.0--1.1.sql pg_stat_statements--1.1--1.2.sql pg_stat_statements--1.2--1.3.sql pg_stat_statements--1.3--1.4.sql pg_stat_statements--1.4--1.5.sql pg_stat_statements--1.4.sql pg_stat_statements--1.5--1.6.sql"; \ + # the first loop is for pg_stat_statement extension version <= 1.6 + for file in /usr/local/pgsql/share/extension/pg_stat_statements--*.sql; do \ + filename=$(basename "$file"); \ + if echo "$old_list" | grep -q -F "$filename"; then \ + echo 'GRANT EXECUTE ON FUNCTION pg_stat_statements_reset() TO neon_superuser;' >> $file; \ + fi; \ + done; \ + # the second loop is for pg_stat_statement extension versions >= 1.7, + # where pg_stat_statement_reset() got 3 additional arguments + for file in /usr/local/pgsql/share/extension/pg_stat_statements--*.sql; do \ + filename=$(basename "$file"); \ + if ! echo "$old_list" | grep -q -F "$filename"; then \ + echo 'GRANT EXECUTE ON FUNCTION pg_stat_statements_reset(Oid, Oid, bigint) TO neon_superuser;' >> $file; \ + fi; \ + done + +######################################################################################### +# +# Layer "neon-pg-ext-build" +# compile neon extensions +# +######################################################################################### +FROM build-deps AS neon-pg-ext-build +ARG PG_VERSION + +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY pgxn/ pgxn/ + +RUN make -j $(getconf _NPROCESSORS_ONLN) \ + PG_CONFIG=/usr/local/pgsql/bin/pg_config \ + -C pgxn/neon \ + -s install && \ + make -j $(getconf _NPROCESSORS_ONLN) \ + PG_CONFIG=/usr/local/pgsql/bin/pg_config \ + -C pgxn/neon_utils \ + -s install && \ + make -j $(getconf _NPROCESSORS_ONLN) \ + PG_CONFIG=/usr/local/pgsql/bin/pg_config \ + -C pgxn/neon_test_utils \ + -s install && \ + make -j $(getconf _NPROCESSORS_ONLN) \ + PG_CONFIG=/usr/local/pgsql/bin/pg_config \ + -C pgxn/neon_rmgr \ + -s install && \ + case "${PG_VERSION}" in \ + "v14" | "v15") \ + ;; \ + "v16") \ + echo "Skipping HNSW for PostgreSQL 16" && exit 0 \ + ;; \ + *) \ + echo "unexpected PostgreSQL version" && exit 1 \ + ;; \ + esac && \ + make -j $(getconf _NPROCESSORS_ONLN) \ + PG_CONFIG=/usr/local/pgsql/bin/pg_config \ + -C pgxn/hnsw \ + -s install + +######################################################################################### +# +# Compile and run the Neon-specific `compute_ctl` binary +# +######################################################################################### +FROM $REPOSITORY/$IMAGE:$TAG AS compute-tools +ARG BUILD_TAG +ENV BUILD_TAG=$BUILD_TAG + +USER nonroot +# Copy entire project to get Cargo.* files with proper dependencies for the whole project +COPY --chown=nonroot . . +RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto + +######################################################################################### +# +# Clean up postgres folder before inclusion +# +######################################################################################### +FROM neon-pg-ext-build AS postgres-cleanup-layer +COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql + +# Remove binaries from /bin/ that we won't use (or would manually copy & install otherwise) +RUN cd /usr/local/pgsql/bin && rm ecpg + +# Remove headers that we won't need anymore - we've completed installation of all extensions +RUN rm -r /usr/local/pgsql/include + +# Remove static postgresql libraries - all compilation is finished, so we +# can now remove these files - they must be included in other binaries by now +# if they were to be used by other libraries. +RUN rm /usr/local/pgsql/lib/lib*.a + +######################################################################################### +# +# Final layer +# Put it all together into the final image +# +######################################################################################### +FROM debian:bullseye-slim + +COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local/pgsql