diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6ba4ff7186..3ff76794bb 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -737,34 +737,6 @@ jobs: --destination neondatabase/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} --cleanup - # Due to a kaniko bug, we can't use cache for extensions image, thus it takes about the same amount of time as compute-node image to build (~10 min) - # During the transition period we need to have extensions in both places (in S3 and in compute-node image), - # so we won't build extension twice, but extract them from compute-node. - # - # For now we use extensions image only for new custom extensitons - - name: Kaniko build extensions only - run: | - # Kaniko is suposed to clean up after itself if --cleanup flag is set, but it doesn't. - # Despite some fixes were made in https://github.com/GoogleContainerTools/kaniko/pull/2504 (in kaniko v1.11.0), - # it still fails with error: - # error building image: could not save file: copying file: symlink postgres /kaniko/1/usr/local/pgsql/bin/postmaster: file exists - # - # Ref https://github.com/GoogleContainerTools/kaniko/issues/1406 - find /kaniko -maxdepth 1 -mindepth 1 -type d -regex "/kaniko/[0-9]*" -exec rm -rv {} \; - - /kaniko/executor --reproducible --snapshot-mode=redo --skip-unused-stages --cache=true \ - --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache \ - --context . \ - --build-arg GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} \ - --build-arg PG_VERSION=${{ matrix.version }} \ - --build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}} \ - --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com \ - --dockerfile Dockerfile.compute-node \ - --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ - --destination neondatabase/extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ - --cleanup \ - --target postgres-extensions - # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ecr': Permission denied - name: Cleanup ECR folder run: rm -rf ~/.ecr @@ -973,57 +945,10 @@ jobs: } }" - upload-postgres-extensions-to-s3: - if: | - (github.ref_name == 'main' || github.ref_name == 'release') && - github.event_name != 'workflow_dispatch' - runs-on: ${{ github.ref_name == 'release' && fromJSON('["self-hosted", "prod", "x64"]') || fromJSON('["self-hosted", "gen3", "small"]') }} - needs: [ tag, promote-images ] - strategy: - fail-fast: false - matrix: - version: [ v14, v15 ] - - env: - EXTENSIONS_IMAGE: ${{ github.ref_name == 'release' && '093970136003' || '369495373322'}}.dkr.ecr.eu-central-1.amazonaws.com/extensions-${{ matrix.version }}:${{ github.ref_name == 'release' && 'latest' || needs.tag.outputs.build-tag }} - AWS_ACCESS_KEY_ID: ${{ github.ref_name == 'release' && secrets.AWS_ACCESS_KEY_PROD || secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ github.ref_name == 'release' && secrets.AWS_SECRET_KEY_PROD || secrets.AWS_SECRET_KEY_DEV }} - S3_BUCKETS: ${{ github.ref_name == 'release' && vars.S3_EXTENSIONS_BUCKETS_PROD || vars.S3_EXTENSIONS_BUCKETS_DEV }} - - steps: - - name: Pull postgres-extensions image - run: | - docker pull ${EXTENSIONS_IMAGE} - - - name: Create postgres-extensions container - id: create-container - run: | - EID=$(docker create ${EXTENSIONS_IMAGE} true) - echo "EID=${EID}" >> $GITHUB_OUTPUT - - - name: Extract postgres-extensions from container - run: | - rm -rf ./extensions-to-upload # Just in case - mkdir -p extensions-to-upload - - docker cp ${{ steps.create-container.outputs.EID }}:/extensions/ ./extensions-to-upload/ - docker cp ${{ steps.create-container.outputs.EID }}:/ext_index.json ./extensions-to-upload/ - - - name: Upload postgres-extensions to S3 - run: | - for BUCKET in $(echo ${S3_BUCKETS:-[]} | jq --raw-output '.[]'); do - aws s3 cp --recursive --only-show-errors ./extensions-to-upload s3://${BUCKET}/${{ needs.tag.outputs.build-tag }}/${{ matrix.version }} - done - - - name: Cleanup - if: ${{ always() && steps.create-container.outputs.EID }} - run: | - docker rm ${{ steps.create-container.outputs.EID }} || true - deploy: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - needs: [ upload-postgres-extensions-to-s3, promote-images, tag, regress-tests ] + needs: [ promote-images, tag, regress-tests ] if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch' steps: - name: Fix git ownership diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 6dc6612e93..18a75e544a 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -764,29 +764,6 @@ RUN rm -r /usr/local/pgsql/include # if they were to be used by other libraries. RUN rm /usr/local/pgsql/lib/lib*.a -######################################################################################### -# -# Extenstion only -# -######################################################################################### -FROM python:3.9-slim-bullseye AS generate-ext-index -ARG PG_VERSION -ARG BUILD_TAG -RUN apt update && apt install -y zstd - -# copy the control files here -COPY --from=kq-imcx-pg-build /extensions/ /extensions/ -COPY --from=pg-anon-pg-build /extensions/ /extensions/ -COPY --from=postgis-build /extensions/ /extensions/ -COPY scripts/combine_control_files.py ./combine_control_files.py -RUN python3 ./combine_control_files.py ${PG_VERSION} ${BUILD_TAG} --public_extensions="anon,postgis" - -FROM scratch AS postgres-extensions -# After the transition this layer will include all extensitons. -# As for now, it's only a couple for testing purposses -COPY --from=generate-ext-index /extensions/*.tar.zst /extensions/ -COPY --from=generate-ext-index /ext_index.json /ext_index.json - ######################################################################################### # # Final layer diff --git a/scripts/combine_control_files.py b/scripts/combine_control_files.py deleted file mode 100644 index b1fe73881e..0000000000 --- a/scripts/combine_control_files.py +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/env python3 -# Script to generate ext_index.json metadata file -# that stores content of the control files and location of extension archives -# for all extensions in extensions subdir. -import argparse -import json -import subprocess -from pathlib import Path - -""" -# ext_index.json example: -{ - "public_extensions": [ - "anon" - ], - "library_index": { - "anon": "anon", - // for more complex extensions like postgis - // we might have something like: - // address_standardizer: postgis - // postgis_tiger: postgis - }, - "extension_data": { - "anon": { - "control_data": { - "anon.control": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n" - }, - "archive_path": "5648391853/v15/extensions/anon.tar.zst" - } - } -} -""" - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="generate ext_index.json") - parser.add_argument("pg_version", type=str, choices=["v14", "v15"], help="pg_version") - parser.add_argument("BUILD_TAG", type=str, help="BUILD_TAG for this compute image") - parser.add_argument("--public_extensions", type=str, help="list of public extensions") - args = parser.parse_args() - pg_version = args.pg_version - BUILD_TAG = args.BUILD_TAG - public_ext_list = args.public_extensions.split(",") - - ext_index = {} - library_index = {} - EXT_PATH = Path("extensions") - for extension in EXT_PATH.iterdir(): - if extension.is_dir(): - control_data = {} - for control_file in extension.glob("*.control"): - if control_file.suffix != ".control": - continue - with open(control_file, "r") as f: - control_data[control_file.name] = f.read() - ext_index[extension.name] = { - "control_data": control_data, - "archive_path": f"{BUILD_TAG}/{pg_version}/extensions/{extension.name}.tar.zst", - } - elif extension.suffix == ".zst": - file_list = ( - str(subprocess.check_output(["tar", "tf", str(extension)]), "utf-8") - .strip() - .split("\n") - ) - for file in file_list: - if file.endswith(".so") and file.startswith("lib/"): - lib_name = file[4:-3] - library_index[lib_name] = extension.name.replace(".tar.zst", "") - - all_data = { - "public_extensions": public_ext_list, - "library_index": library_index, - "extension_data": ext_index, - } - with open("ext_index.json", "w") as f: - json.dump(all_data, f)