diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 918e568e27..1088f67710 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -583,7 +583,7 @@ jobs: run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - name: Kaniko build compute node with extensions - run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --dockerfile Dockerfile.compute-node-${{ matrix.version }} --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} + run: /kaniko/executor --skip-unused-stages --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} --build-arg PG_VERSION=${{ matrix.version }} --dockerfile Dockerfile.compute-node --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} vm-compute-node-image: runs-on: [ self-hosted, dev, x64 ] diff --git a/Dockerfile.compute-node-v14 b/Dockerfile.compute-node similarity index 98% rename from Dockerfile.compute-node-v14 rename to Dockerfile.compute-node index fb115ca641..3d6cfd4e6b 100644 --- a/Dockerfile.compute-node-v14 +++ b/Dockerfile.compute-node @@ -1,8 +1,3 @@ -# -# This file is identical to the Dockerfile.compute-node-v15 file -# except for the version of Postgres that is built. -# - ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com ARG IMAGE=rust ARG TAG=pinned @@ -24,7 +19,8 @@ RUN apt update && \ # ######################################################################################### FROM build-deps AS pg-build -COPY vendor/postgres-v14 postgres +ARG PG_VERSION +COPY vendor/postgres-${PG_VERSION} postgres RUN cd postgres && \ ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \ make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ diff --git a/Dockerfile.compute-node-v15 b/Dockerfile.compute-node-v15 deleted file mode 100644 index efbe0c6872..0000000000 --- a/Dockerfile.compute-node-v15 +++ /dev/null @@ -1,239 +0,0 @@ -# -# This file is identical to the Dockerfile.compute-node-v14 file -# except for the version of Postgres that is built. -# - -ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com -ARG IMAGE=rust -ARG TAG=pinned - -######################################################################################### -# -# Layer "build-deps" -# -######################################################################################### -FROM debian:bullseye-slim AS build-deps -RUN apt update && \ - apt install -y git autoconf automake libtool build-essential bison flex libreadline-dev \ - zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget pkg-config libssl-dev - -######################################################################################### -# -# Layer "pg-build" -# Build Postgres from the neon postgres repository. -# -######################################################################################### -FROM build-deps AS pg-build -COPY vendor/postgres-v15 postgres -RUN cd postgres && \ - ./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp && \ - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s install && \ - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \ - # Install headers - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \ - make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install && \ - # Enable some of contrib extensions - echo 'trusted = true' >> /usr/local/pgsql/share/extension/bloom.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrowlocks.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/intagg.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgstattuple.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control - -######################################################################################### -# -# Layer "postgis-build" -# Build PostGIS from the upstream PostGIS mirror. -# -######################################################################################### -FROM build-deps AS postgis-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ -RUN apt update && \ - apt install -y gdal-bin libgdal-dev libprotobuf-c-dev protobuf-c-compiler xsltproc - -RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \ - tar xvzf postgis-3.3.1.tar.gz && \ - cd postgis-3.3.1 && \ - ./autogen.sh && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ - ./configure && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ - cd extensions/postgis && \ - make clean && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_raster.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer_data_us.control - -######################################################################################### -# -# Layer "plv8-build" -# Build plv8 -# -######################################################################################### -FROM build-deps AS plv8-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ -RUN apt update && \ - apt install -y ninja-build python3-dev libc++-dev libc++abi-dev libncurses5 binutils - -# https://github.com/plv8/plv8/issues/475: -# v8 uses gold for linking and sets `--thread-count=4` which breaks -# gold version <= 1.35 (https://sourceware.org/bugzilla/show_bug.cgi?id=23607) -# Install newer gold version manually as debian-testing binutils version updates -# libc version, which in turn breaks other extension built against non-testing libc. -RUN wget https://ftp.gnu.org/gnu/binutils/binutils-2.38.tar.gz && \ - tar xvzf binutils-2.38.tar.gz && \ - cd binutils-2.38 && \ - cd libiberty && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && \ - cd ../bfd && ./configure && make bfdver.h && \ - cd ../gold && ./configure && make -j $(getconf _NPROCESSORS_ONLN) && make install && \ - cp /usr/local/bin/ld.gold /usr/bin/gold - -# Sed is used to patch for https://github.com/plv8/plv8/issues/503 -RUN wget https://github.com/plv8/plv8/archive/refs/tags/v3.1.4.tar.gz && \ - tar xvzf v3.1.4.tar.gz && \ - cd plv8-3.1.4 && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ - sed -i 's/MemoryContextAlloc(/MemoryContextAllocZero(/' plv8.cc && \ - make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \ - rm -rf /plv8-* && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/plv8.control - -######################################################################################### -# -# Layer "h3-pg-build" -# Build h3_pg -# -######################################################################################### -FROM build-deps AS h3-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - -# packaged cmake is too old -RUN wget https://github.com/Kitware/CMake/releases/download/v3.24.2/cmake-3.24.2-linux-x86_64.sh \ - -q -O /tmp/cmake-install.sh \ - && chmod u+x /tmp/cmake-install.sh \ - && /tmp/cmake-install.sh --skip-license --prefix=/usr/local/ \ - && rm /tmp/cmake-install.sh - -RUN wget https://github.com/uber/h3/archive/refs/tags/v4.0.1.tar.gz -O h3.tgz && \ - tar xvzf h3.tgz && \ - cd h3-4.0.1 && \ - mkdir build && \ - cd build && \ - cmake .. -DCMAKE_BUILD_TYPE=Release && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - DESTDIR=/h3 make install && \ - cp -R /h3/usr / && \ - rm -rf build - -RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.0.1.tar.gz -O h3-pg.tgz && \ - tar xvzf h3-pg.tgz && \ - cd h3-pg-4.0.1 && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3_postgis.control - -######################################################################################### -# -# Layer "unit-pg-build" -# compile unit extension -# -######################################################################################### -FROM build-deps AS unit-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - -RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz && \ - tar xvzf 7.7.tar.gz && \ - cd postgresql-unit-7.7 && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/unit.control - -######################################################################################### -# -# Layer "neon-pg-ext-build" -# compile neon extensions -# -######################################################################################### -FROM build-deps AS neon-pg-ext-build -COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY --from=h3-pg-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY --from=h3-pg-build /h3/usr / -COPY --from=unit-pg-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY pgxn/ pgxn/ - -RUN make -j $(getconf _NPROCESSORS_ONLN) \ - PG_CONFIG=/usr/local/pgsql/bin/pg_config \ - -C pgxn/neon \ - -s install - -######################################################################################### -# -# Compile and run the Neon-specific `compute_ctl` binary -# -######################################################################################### -FROM $REPOSITORY/$IMAGE:$TAG AS compute-tools -USER nonroot -# Copy entire project to get Cargo.* files with proper dependencies for the whole project -COPY --chown=nonroot . . -RUN cd compute_tools && cargo build --locked --profile release-line-debug-size-lto - -######################################################################################### -# -# Clean up postgres folder before inclusion -# -######################################################################################### -FROM neon-pg-ext-build AS postgres-cleanup-layer -COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql - -# Remove binaries from /bin/ that we won't use (or would manually copy & install otherwise) -RUN cd /usr/local/pgsql/bin && rm ecpg raster2pgsql shp2pgsql pgtopo_export pgtopo_import pgsql2shp - -# Remove headers that we won't need anymore - we've completed installation of all extensions -RUN rm -r /usr/local/pgsql/include - -# Remove static postgresql libraries - all compilation is finished, so we -# can now remove these files - they must be included in other binaries by now -# if they were to be used by other libraries. -RUN rm /usr/local/pgsql/lib/lib*.a - -######################################################################################### -# -# Final layer -# Put it all together into the final image -# -######################################################################################### -FROM debian:bullseye-slim -# Add user postgres -RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \ - echo "postgres:test_console_pass" | chpasswd && \ - mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \ - chown -R postgres:postgres /var/db/postgres && \ - chmod 0750 /var/db/postgres/compute && \ - echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig - -COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local -COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl - -# Install: -# libreadline8 for psql -# libossp-uuid16 for extension ossp-uuid -# libgeos, libgdal, libproj and libprotobuf-c1 for PostGIS -RUN apt update && \ - apt install --no-install-recommends -y \ - libreadline8 \ - libossp-uuid16 \ - libgeos-c1v5 \ - libgdal28 \ - libproj19 \ - libprotobuf-c1 \ - gdb && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -USER postgres -ENTRYPOINT ["/usr/local/bin/compute_ctl"]