From ed95dc1c7a36ba8dad533f59850a4cd4184a6255 Mon Sep 17 00:00:00 2001 From: Alek Westover Date: Mon, 10 Jul 2023 11:04:10 -0400 Subject: [PATCH] try to do it with just copy-pasting a lot of code --- .github/workflows/build_and_test.yml | 30 ++++++++++------------------ Dockerfile.compute-node | 27 ++++++++++++++----------- 2 files changed, 25 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1630d039cf..3d3e0b8511 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -956,29 +956,19 @@ jobs: - name: Extract postgres-extensions from container run: | - rm -rf ./extensions-to-upload ./custom-extensions # Just in case + rm -rf ./extensions-to-upload # Just in case + mkdir extensions-to-upload - # In compute image we have a bit different directory layout - mkdir -p extensions-to-upload/share - docker cp ${{ steps.create-container.outputs.CID }}:/usr/local/share/extension/*.tar.gz ./extensions-to-upload/share/extension - docker cp ${{ steps.create-container.outputs.CID }}:/usr/local/share/extension/*.control ./extensions-to-upload/share/extension - docker cp ${{ steps.create-container.outputs.CID }}:/usr/local/lib/*.tar.gz ./extensions-to-upload/lib + # TODO: Delete Neon extensitons (they always present on compute-node image) + # rm -rf ./extensions-to-upload/share/extension/neon* + # rm -rf ./extensions-to-upload/lib/neon* - # Delete Neon extensitons (they always present on compute-node image) - rm -rf ./extensions-to-upload/share/extension/neon* - rm -rf ./extensions-to-upload/lib/neon* + # # TODO: Delete leftovers from the extension build step + # # (This step, if desired, should be moved to Dockerfile.compute-node before the files are zipped) + # rm -rf ./extensions-to-upload/lib/pgxs + # rm -rf ./extensions-to-upload/lib/pkgconfig - # Delete leftovers from the extension build step - rm -rf ./extensions-to-upload/lib/pgxs - rm -rf ./extensions-to-upload/lib/pkgconfig - - docker cp ${{ steps.create-container.outputs.EID }}:/extensions ./custom-extensions - for EXT_NAME in $(ls ./custom-extensions); do - mkdir -p ./extensions-to-upload/${EXT_NAME}/share - - mv ./custom-extensions/${EXT_NAME}/share/extension ./extensions-to-upload/${EXT_NAME}/share/extension - mv ./custom-extensions/${EXT_NAME}/lib ./extensions-to-upload/${EXT_NAME}/lib - done + docker cp -r ${{ steps.create-container.outputs.EID }}:/extensions ./extensions-to-upload - name: Upload postgres-extensions to S3 run: | diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index b037f90d4a..b2c6fc2844 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -530,9 +530,14 @@ ENV PATH "/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/neondatabase/pg_embedding/archive/2465f831ea1f8d49c1d74f8959adb7fc277d70cd.tar.gz -O pg_embedding.tar.gz && \ echo "047af2b1f664a1e6e37867bd4eeaf5934fa27d6ba3d6c4461efa388ddf7cd1d5 pg_embedding.tar.gz" | sha256sum --check && \ mkdir pg_embedding-src && cd pg_embedding-src && tar xvzf ../pg_embedding.tar.gz --strip-components=1 -C . && \ + find /usr/local/pgsql -type f | sort > /before_embedding.txt && \ make -j $(getconf _NPROCESSORS_ONLN) && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/embedding.control + echo 'trusted = true' >> /usr/local/pgsql/share/extension/embedding.control &&\ + find /usr/local/pgsql -type f | sort > /after_embedding.txt &&\ + /bin/bash -c 'for from in $(comm -13 /before_embedding.txt /after_embedding.txt); do to=/extensions/public/embedding/${from:17} && mkdir -p $(dirname ${to}) && cp -a ${from} ${to}; done' && \ + tar -zcvf /extensions/public/embedding.tar.gz /extensions/public/embedding && \ + cp /usr/local/pgsql/share/extension/embedding.control /control_files/embedding.control ######################################################################################### # @@ -552,10 +557,9 @@ RUN wget https://gitlab.com/dalibo/postgresql_anonymizer/-/archive/1.1.0/postgre make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/anon.control && \ find /usr/local/pgsql -type f | sort > /after.txt && \ - /bin/bash -c 'for from in $(comm -13 /before.txt /after.txt); do to=/extensions/anon/${from:17} && mkdir -p $(dirname ${to}) && cp -a ${from} ${to}; done' && \ - tar -zcvf /extensions/anon/share/extension/anon.tar.gz /extensions/anon/share/extension && \ - tar -zcvf /extensions/anon/lib/anon.tar.gz /extensions/anon/lib && \ - cp /usr/local/pgsql/share/extension/anon.control /extensions/anon/share/extension/anon.control + /bin/bash -c 'for from in $(comm -13 /before.txt /after.txt); do to=/extensions/123454321/anon/${from:17} && mkdir -p $(dirname ${to}) && cp -a ${from} ${to}; done' && \ + tar -zcvf /extensions/123454321/anon.tar.gz /extensions/123454321/anon && \ + cp /usr/local/pgsql/share/extension/anon.control /control_files/anon.control ######################################################################################### # @@ -751,13 +755,12 @@ FROM scratch AS postgres-extensions # After the transition this layer will include all extensitons. # As for now, it's only for new custom ones # -# # Default extensions -# COPY --from=postgres-cleanup-layer /usr/local/pgsql/share/extension /usr/local/pgsql/share/extension -# COPY --from=postgres-cleanup-layer /usr/local/pgsql/lib /usr/local/pgsql/lib -# Custom extensions -COPY --from=pg-anon-pg-build /extensions/anon/lib/anon.tar.gz /extensions/anon/lib/anon.tar.gz -COPY --from=pg-anon-pg-build /extensions/anon/share/extension/anon.tar.gz /extensions/anon/share/extension/anon.tar.gz -COPY --from=pg-anon-pg-build /extensions/anon/share/extension/anon.control /extensions/anon/share/extension/anon.control +# extensions +COPY --from=pg-anon-pg-build /extensions/123456789/anon.tar.gz /extensions/123456789/anon.tar.gz +COPY --from=pg-anon-pg-build /control_files/anon.control /control_files/anon.control + +COPY --from=pg-embedding-pg-build /extensions/public/embedding.tar.gz /extensions/public/embedding.tar.gz +COPY --from=pg-embedding-pg-build /control_files/embedding.control /control_files/embedding.control ######################################################################################### #