From 1d5e476c961cb53089f9eebbd8d67c9902611232 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Wed, 28 Feb 2024 13:38:11 +0100 Subject: [PATCH] CI: use build-tools image from dockerhub (#6795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Currently, after updating `Dockerfile.build-tools` in a PR, it requires a manual action to make it `pinned`, i.e., the default for everyone. It also makes all opened PRs use such images (even created in the PR and without such changes). This PR overhauls the way we build and use `build-tools` image (and uses the image from Docker Hub). ## Summary of changes - The `neondatabase/build-tools` image gets tagged with the latest commit sha for the `Dockerfile.build-tools` file - Each PR calculates the tag for `neondatabase/build-tools`, tries to pull it, and rebuilds the image with such tag if it doesn't exist. - Use `neondatabase/build-tools` as a default image - When running on `main` branch — create a `pinned` tag and push it to ECR - Use `concurrency` to ensure we don't build `build-tools` image for the same commit in parallel from different PRs --- .github/workflows/benchmarking.yml | 10 +- .github/workflows/build-build-tools-image.yml | 105 +++++++++++++++ .../workflows/build_and_push_docker_image.yml | 124 ------------------ .github/workflows/build_and_test.yml | 91 ++++++++----- .github/workflows/check-build-tools-image.yml | 58 ++++++++ .github/workflows/neon_extra_builds.yml | 32 ++++- .github/workflows/pin-build-tools-image.yml | 72 ++++++++++ .../workflows/update_build_tools_image.yml | 70 ---------- CONTRIBUTING.md | 15 +-- ...rfile.buildtools => Dockerfile.build-tools | 0 10 files changed, 332 insertions(+), 245 deletions(-) create mode 100644 .github/workflows/build-build-tools-image.yml delete mode 100644 .github/workflows/build_and_push_docker_image.yml create mode 100644 .github/workflows/check-build-tools-image.yml create mode 100644 .github/workflows/pin-build-tools-image.yml delete mode 100644 .github/workflows/update_build_tools_image.yml rename Dockerfile.buildtools => Dockerfile.build-tools (100%) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index fc245f42a8..2e56bf909f 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -62,7 +62,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned options: --init steps: @@ -214,7 +214,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned options: --init # Increase timeout to 8h, default timeout is 6h @@ -362,7 +362,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned options: --init steps: @@ -461,7 +461,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned options: --init steps: @@ -558,7 +558,7 @@ jobs: runs-on: [ self-hosted, us-east-2, x64 ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned options: --init steps: diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml new file mode 100644 index 0000000000..251423e701 --- /dev/null +++ b/.github/workflows/build-build-tools-image.yml @@ -0,0 +1,105 @@ +name: Build build-tools image + +on: + workflow_call: + inputs: + image-tag: + description: "build-tools image tag" + required: true + type: string + outputs: + image-tag: + description: "build-tools tag" + value: ${{ inputs.image-tag }} + image: + description: "build-tools image" + value: neondatabase/build-tools:${{ inputs.image-tag }} + +defaults: + run: + shell: bash -euo pipefail {0} + +concurrency: + group: build-build-tools-image-${{ inputs.image-tag }} + +# No permission for GITHUB_TOKEN by default; the **minimal required** set of permissions should be granted in each job. +permissions: {} + +jobs: + check-image: + uses: ./.github/workflows/check-build-tools-image.yml + + # This job uses older version of GitHub Actions because it's run on gen2 runners, which don't support node 20 (for newer versions) + build-image: + needs: [ check-image ] + if: needs.check-image.outputs.found == 'false' + + strategy: + matrix: + arch: [ x64, arm64 ] + + runs-on: ${{ fromJson(format('["self-hosted", "dev", "{0}"]', matrix.arch)) }} + + env: + IMAGE_TAG: ${{ inputs.image-tag }} + + steps: + - name: Check `input.tag` is correct + env: + INPUTS_IMAGE_TAG: ${{ inputs.image-tag }} + CHECK_IMAGE_TAG : ${{ needs.check-image.outputs.image-tag }} + run: | + if [ "${INPUTS_IMAGE_TAG}" != "${CHECK_IMAGE_TAG}" ]; then + echo "'inputs.image-tag' (${INPUTS_IMAGE_TAG}) does not match the tag of the latest build-tools image 'inputs.image-tag' (${CHECK_IMAGE_TAG})" + exit 1 + fi + + - uses: actions/checkout@v3 + + # Use custom DOCKER_CONFIG directory to avoid conflicts with default settings + # The default value is ~/.docker + - name: Set custom docker config directory + run: | + mkdir -p /tmp/.docker-custom + echo DOCKER_CONFIG=/tmp/.docker-custom >> $GITHUB_ENV + + - uses: docker/setup-buildx-action@v2 + + - uses: docker/login-action@v2 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + + - uses: docker/build-push-action@v4 + with: + context: . + provenance: false + push: true + pull: true + file: Dockerfile.build-tools + cache-from: type=registry,ref=neondatabase/build-tools:cache-${{ matrix.arch }} + cache-to: type=registry,ref=neondatabase/build-tools:cache-${{ matrix.arch }},mode=max + tags: neondatabase/build-tools:${{ inputs.image-tag }}-${{ matrix.arch }} + + - name: Remove custom docker config directory + run: | + rm -rf /tmp/.docker-custom + + merge-images: + needs: [ build-image ] + runs-on: ubuntu-latest + + env: + IMAGE_TAG: ${{ inputs.image-tag }} + + steps: + - uses: docker/login-action@v3 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + + - name: Create multi-arch image + run: | + docker buildx imagetools create -t neondatabase/build-tools:${IMAGE_TAG} \ + neondatabase/build-tools:${IMAGE_TAG}-x64 \ + neondatabase/build-tools:${IMAGE_TAG}-arm64 diff --git a/.github/workflows/build_and_push_docker_image.yml b/.github/workflows/build_and_push_docker_image.yml deleted file mode 100644 index 892e21114b..0000000000 --- a/.github/workflows/build_and_push_docker_image.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Build and Push Docker Image - -on: - workflow_call: - inputs: - dockerfile-path: - required: true - type: string - image-name: - required: true - type: string - outputs: - build-tools-tag: - description: "tag generated for build tools" - value: ${{ jobs.tag.outputs.build-tools-tag }} - -jobs: - check-if-build-tools-dockerfile-changed: - runs-on: ubuntu-latest - outputs: - docker_file_changed: ${{ steps.dockerfile.outputs.docker_file_changed }} - steps: - - name: Check if Dockerfile.buildtools has changed - id: dockerfile - run: | - if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then - echo "docker_file_changed=false" >> $GITHUB_OUTPUT - exit - fi - updated_files=$(gh pr --repo neondatabase/neon diff ${{ github.event.pull_request.number }} --name-only) - if [[ $updated_files == *"Dockerfile.buildtools"* ]]; then - echo "docker_file_changed=true" >> $GITHUB_OUTPUT - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - tag: - runs-on: ubuntu-latest - needs: [ check-if-build-tools-dockerfile-changed ] - outputs: - build-tools-tag: ${{steps.buildtools-tag.outputs.image_tag}} - - steps: - - name: Get buildtools tag - env: - DOCKERFILE_CHANGED: ${{ needs.check-if-build-tools-dockerfile-changed.outputs.docker_file_changed }} - run: | - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] && [[ "${DOCKERFILE_CHANGED}" == "true" ]]; then - IMAGE_TAG=$GITHUB_RUN_ID - else - IMAGE_TAG=pinned - fi - - echo "image_tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT - shell: bash - id: buildtools-tag - - kaniko: - if: needs.check-if-build-tools-dockerfile-changed.outputs.docker_file_changed == 'true' - needs: [ tag, check-if-build-tools-dockerfile-changed ] - runs-on: [ self-hosted, dev, x64 ] - container: gcr.io/kaniko-project/executor:v1.7.0-debug - - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Configure ECR login - run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - - - name: Kaniko build - run: | - /kaniko/executor \ - --reproducible \ - --snapshotMode=redo \ - --skip-unused-stages \ - --dockerfile ${{ inputs.dockerfile-path }} \ - --cache=true \ - --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache \ - --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${{ inputs.image-name }}:${{ needs.tag.outputs.build-tools-tag }}-amd64 - - kaniko-arm: - if: needs.check-if-build-tools-dockerfile-changed.outputs.docker_file_changed == 'true' - needs: [ tag, check-if-build-tools-dockerfile-changed ] - runs-on: [ self-hosted, dev, arm64 ] - container: gcr.io/kaniko-project/executor:v1.7.0-debug - - steps: - - name: Checkout - uses: actions/checkout@v1 - - - name: Configure ECR login - run: echo "{\"credsStore\":\"ecr-login\"}" > /kaniko/.docker/config.json - - - name: Kaniko build - run: | - /kaniko/executor \ - --reproducible \ - --snapshotMode=redo \ - --skip-unused-stages \ - --dockerfile ${{ inputs.dockerfile-path }} \ - --cache=true \ - --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache \ - --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${{ inputs.image-name }}:${{ needs.tag.outputs.build-tools-tag }}-arm64 - - manifest: - if: needs.check-if-build-tools-dockerfile-changed.outputs.docker_file_changed == 'true' - name: 'manifest' - runs-on: [ self-hosted, dev, x64 ] - needs: - - tag - - kaniko - - kaniko-arm - - check-if-build-tools-dockerfile-changed - - steps: - - name: Create manifest - run: | - docker manifest create 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${{ inputs.image-name }}:${{ needs.tag.outputs.build-tools-tag }} \ - --amend 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${{ inputs.image-name }}:${{ needs.tag.outputs.build-tools-tag }}-amd64 \ - --amend 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${{ inputs.image-name }}:${{ needs.tag.outputs.build-tools-tag }}-arm64 - - - name: Push manifest - run: docker manifest push 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${{ inputs.image-name }}:${{ needs.tag.outputs.build-tools-tag }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0e67259b3f..e29a58bbe2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -77,19 +77,25 @@ jobs: shell: bash id: build-tag - build-buildtools-image: + check-build-tools-image: needs: [ check-permissions ] - uses: ./.github/workflows/build_and_push_docker_image.yml + uses: ./.github/workflows/check-build-tools-image.yml + + build-build-tools-image: + needs: [ check-build-tools-image ] + uses: ./.github/workflows/build-build-tools-image.yml with: - dockerfile-path: Dockerfile.buildtools - image-name: build-tools + image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} secrets: inherit check-codestyle-python: - needs: [ check-permissions, build-buildtools-image ] + needs: [ check-permissions, build-build-tools-image ] runs-on: [ self-hosted, gen3, small ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init steps: @@ -118,10 +124,13 @@ jobs: run: poetry run mypy . check-codestyle-rust: - needs: [ check-permissions, build-buildtools-image ] + needs: [ check-permissions, build-build-tools-image ] runs-on: [ self-hosted, gen3, small ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init steps: @@ -185,10 +194,13 @@ jobs: run: cargo deny check --hide-inclusion-graph build-neon: - needs: [ check-permissions, tag, build-buildtools-image ] + needs: [ check-permissions, tag, build-build-tools-image ] runs-on: [ self-hosted, gen3, large ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} # Raise locked memory limit for tokio-epoll-uring. # On 5.10 LTS kernels < 5.10.162 (and generally mainline kernels < 5.12), # io_uring will account the memory of the CQ and SQ as locked. @@ -426,10 +438,13 @@ jobs: uses: ./.github/actions/save-coverage-data regress-tests: - needs: [ check-permissions, build-neon, build-buildtools-image, tag ] + needs: [ check-permissions, build-neon, build-build-tools-image, tag ] runs-on: [ self-hosted, gen3, large ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} # for changed limits, see comments on `options:` earlier in this file options: --init --shm-size=512mb --ulimit memlock=67108864:67108864 strategy: @@ -473,10 +488,13 @@ jobs: get-benchmarks-durations: outputs: json: ${{ steps.get-benchmark-durations.outputs.json }} - needs: [ check-permissions, build-buildtools-image ] + needs: [ check-permissions, build-build-tools-image ] runs-on: [ self-hosted, gen3, small ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') steps: @@ -503,10 +521,13 @@ jobs: echo "json=$(jq --compact-output '.' /tmp/benchmark_durations.json)" >> $GITHUB_OUTPUT benchmarks: - needs: [ check-permissions, build-neon, build-buildtools-image, get-benchmarks-durations ] + needs: [ check-permissions, build-neon, build-build-tools-image, get-benchmarks-durations ] runs-on: [ self-hosted, gen3, small ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} # for changed limits, see comments on `options:` earlier in this file options: --init --shm-size=512mb --ulimit memlock=67108864:67108864 if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') @@ -538,12 +559,15 @@ jobs: # while coverage is currently collected for the debug ones create-test-report: - needs: [ check-permissions, regress-tests, coverage-report, benchmarks, build-buildtools-image ] + needs: [ check-permissions, regress-tests, coverage-report, benchmarks, build-build-tools-image ] if: ${{ !cancelled() && contains(fromJSON('["skipped", "success"]'), needs.check-permissions.result) }} runs-on: [ self-hosted, gen3, small ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init steps: @@ -584,10 +608,13 @@ jobs: }) coverage-report: - needs: [ check-permissions, regress-tests, build-buildtools-image ] + needs: [ check-permissions, regress-tests, build-build-tools-image ] runs-on: [ self-hosted, gen3, small ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${{ needs.build-buildtools-image.outputs.build-tools-tag }} + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init strategy: fail-fast: false @@ -691,7 +718,7 @@ jobs: secrets: inherit neon-image: - needs: [ check-permissions, build-buildtools-image, tag ] + needs: [ check-permissions, build-build-tools-image, tag ] runs-on: [ self-hosted, gen3, large ] steps: @@ -726,8 +753,7 @@ jobs: build-args: | GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} BUILD_TAG=${{ needs.tag.outputs.build-tag }} - TAG=${{ needs.build-buildtools-image.outputs.build-tools-tag }} - REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com + TAG=${{ needs.build-build-tools-image.outputs.image-tag }} provenance: false push: true pull: true @@ -745,7 +771,7 @@ jobs: compute-tools-image: runs-on: [ self-hosted, gen3, large ] - needs: [ check-permissions, build-buildtools-image, tag ] + needs: [ check-permissions, build-build-tools-image, tag ] steps: - name: Checkout @@ -779,8 +805,7 @@ jobs: build-args: | GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} BUILD_TAG=${{needs.tag.outputs.build-tag}} - TAG=${{needs.build-buildtools-image.outputs.build-tools-tag}} - REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com + TAG=${{ needs.build-build-tools-image.outputs.image-tag }} provenance: false push: true pull: true @@ -797,7 +822,7 @@ jobs: rm -rf .docker-custom compute-node-image: - needs: [ check-permissions, build-buildtools-image, tag ] + needs: [ check-permissions, build-build-tools-image, tag ] runs-on: [ self-hosted, gen3, large ] strategy: @@ -844,8 +869,7 @@ jobs: GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} PG_VERSION=${{ matrix.version }} BUILD_TAG=${{needs.tag.outputs.build-tag}} - TAG=${{needs.build-buildtools-image.outputs.build-tools-tag}} - REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com + TAG=${{ needs.build-build-tools-image.outputs.image-tag }} provenance: false push: true pull: true @@ -938,7 +962,7 @@ jobs: - name: Verify docker-compose example timeout-minutes: 20 - run: env REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh + run: env TAG=${{needs.tag.outputs.build-tag}} ./docker-compose/docker_compose_test.sh - name: Print logs and clean up if: always() @@ -1218,3 +1242,10 @@ jobs: time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} s3://${BUCKET}/${PREFIX}/${FILENAME} done + + pin-build-tools-image: + needs: [ build-build-tools-image, promote-images, regress-tests ] + if: github.ref_name == 'main' + uses: ./.github/workflows/pin-build-tools-image.yml + with: + from-tag: ${{ needs.build-build-tools-image.outputs.image-tag }} diff --git a/.github/workflows/check-build-tools-image.yml b/.github/workflows/check-build-tools-image.yml new file mode 100644 index 0000000000..28646dfc19 --- /dev/null +++ b/.github/workflows/check-build-tools-image.yml @@ -0,0 +1,58 @@ +name: Check build-tools image + +on: + workflow_call: + outputs: + image-tag: + description: "build-tools image tag" + value: ${{ jobs.check-image.outputs.tag }} + found: + description: "Whether the image is found in the registry" + value: ${{ jobs.check-image.outputs.found }} + +defaults: + run: + shell: bash -euo pipefail {0} + +# No permission for GITHUB_TOKEN by default; the **minimal required** set of permissions should be granted in each job. +permissions: {} + +jobs: + check-image: + runs-on: ubuntu-latest + outputs: + tag: ${{ steps.get-build-tools-tag.outputs.image-tag }} + found: ${{ steps.check-image.outputs.found }} + + steps: + - name: Get build-tools image tag for the current commit + id: get-build-tools-tag + env: + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + LAST_BUILD_TOOLS_SHA=$( + gh api \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + --method GET \ + --field path=Dockerfile.build-tools \ + --field sha=${COMMIT_SHA} \ + --field per_page=1 \ + --jq ".[0].sha" \ + "/repos/${GITHUB_REPOSITORY}/commits" + ) + echo "image-tag=${LAST_BUILD_TOOLS_SHA}" | tee -a $GITHUB_OUTPUT + + - name: Check if such tag found in the registry + id: check-image + env: + IMAGE_TAG: ${{ steps.get-build-tools-tag.outputs.image-tag }} + run: | + if docker manifest inspect neondatabase/build-tools:${IMAGE_TAG}; then + found=true + else + found=false + fi + + echo "found=${found}" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 1c9763cc00..5a2f9d6645 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -26,6 +26,17 @@ jobs: with: github-event-name: ${{ github.event_name}} + check-build-tools-image: + needs: [ check-permissions ] + uses: ./.github/workflows/check-build-tools-image.yml + + build-build-tools-image: + needs: [ check-build-tools-image ] + uses: ./.github/workflows/build-build-tools-image.yml + with: + image-tag: ${{ needs.check-build-tools-image.outputs.image-tag }} + secrets: inherit + check-macos-build: needs: [ check-permissions ] if: | @@ -123,7 +134,7 @@ jobs: run: ./run_clippy.sh check-linux-arm-build: - needs: [ check-permissions ] + needs: [ check-permissions, build-build-tools-image ] timeout-minutes: 90 runs-on: [ self-hosted, dev, arm64 ] @@ -137,7 +148,10 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:pinned + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init steps: @@ -244,12 +258,15 @@ jobs: cargo nextest run --package remote_storage --test test_real_azure check-codestyle-rust-arm: - needs: [ check-permissions ] + needs: [ check-permissions, build-build-tools-image ] timeout-minutes: 90 runs-on: [ self-hosted, dev, arm64 ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init steps: @@ -316,14 +333,17 @@ jobs: run: cargo deny check gather-rust-build-stats: - needs: [ check-permissions ] + needs: [ check-permissions, build-build-tools-image ] if: | contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') || contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || github.ref_name == 'main' runs-on: [ self-hosted, gen3, large ] container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + image: ${{ needs.build-build-tools-image.outputs.image }} + credentials: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} options: --init env: diff --git a/.github/workflows/pin-build-tools-image.yml b/.github/workflows/pin-build-tools-image.yml new file mode 100644 index 0000000000..c941692066 --- /dev/null +++ b/.github/workflows/pin-build-tools-image.yml @@ -0,0 +1,72 @@ +name: 'Pin build-tools image' + +on: + workflow_dispatch: + inputs: + from-tag: + description: 'Source tag' + required: true + type: string + workflow_call: + inputs: + from-tag: + description: 'Source tag' + required: true + type: string + +defaults: + run: + shell: bash -euo pipefail {0} + +concurrency: + group: pin-build-tools-image-${{ inputs.from-tag }} + +permissions: {} + +jobs: + tag-image: + runs-on: ubuntu-latest + + env: + FROM_TAG: ${{ inputs.from-tag }} + TO_TAG: pinned + + steps: + - name: Check if we really need to pin the image + id: check-manifests + run: | + docker manifest inspect neondatabase/build-tools:${FROM_TAG} > ${FROM_TAG}.json + docker manifest inspect neondatabase/build-tools:${TO_TAG} > ${TO_TAG}.json + + if diff ${FROM_TAG}.json ${TO_TAG}.json; then + skip=true + else + skip=false + fi + + echo "skip=${skip}" | tee -a $GITHUB_OUTPUT + + - uses: docker/login-action@v3 + if: steps.check-manifests.outputs.skip == 'false' + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + + - name: Tag build-tools with `${{ env.TO_TAG }}` in Docker Hub + if: steps.check-manifests.outputs.skip == 'false' + run: | + docker buildx imagetools create -t neondatabase/build-tools:${TO_TAG} \ + neondatabase/build-tools:${FROM_TAG} + + - uses: docker/login-action@v3 + if: steps.check-manifests.outputs.skip == 'false' + with: + registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com + username: ${{ secrets.AWS_ACCESS_KEY_DEV }} + password: ${{ secrets.AWS_SECRET_KEY_DEV }} + + - name: Tag build-tools with `${{ env.TO_TAG }}` in ECR + if: steps.check-manifests.outputs.skip == 'false' + run: | + docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${TO_TAG} \ + neondatabase/build-tools:${FROM_TAG} diff --git a/.github/workflows/update_build_tools_image.yml b/.github/workflows/update_build_tools_image.yml deleted file mode 100644 index 900724fc60..0000000000 --- a/.github/workflows/update_build_tools_image.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: 'Update build tools image tag' - -# This workflow it used to update tag of build tools in ECR. -# The most common use case is adding/moving `pinned` tag to `${GITHUB_RUN_IT}` image. - -on: - workflow_dispatch: - inputs: - from-tag: - description: 'Source tag' - required: true - type: string - to-tag: - description: 'Destination tag' - required: true - type: string - default: 'pinned' - -defaults: - run: - shell: bash -euo pipefail {0} - -permissions: {} - -jobs: - tag-image: - runs-on: [ self-hosted, gen3, small ] - - env: - ECR_IMAGE: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools - DOCKER_HUB_IMAGE: docker.io/neondatabase/build-tools - FROM_TAG: ${{ inputs.from-tag }} - TO_TAG: ${{ inputs.to-tag }} - - steps: - # Use custom DOCKER_CONFIG directory to avoid conflicts with default settings - # The default value is ~/.docker - - name: Set custom docker config directory - run: | - mkdir -p .docker-custom - echo DOCKER_CONFIG=$(pwd)/.docker-custom >> $GITHUB_ENV - - - uses: docker/login-action@v2 - with: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - - uses: docker/login-action@v2 - with: - registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_DEV }} - password: ${{ secrets.AWS_SECRET_KEY_DEV }} - - - uses: actions/setup-go@v5 - with: - go-version: '1.21' - - - name: Install crane - run: | - go install github.com/google/go-containerregistry/cmd/crane@a0658aa1d0cc7a7f1bcc4a3af9155335b6943f40 # v0.18.0 - - - name: Copy images - run: | - crane copy "${ECR_IMAGE}:${FROM_TAG}" "${ECR_IMAGE}:${TO_TAG}" - crane copy "${ECR_IMAGE}:${FROM_TAG}" "${DOCKER_HUB_IMAGE}:${TO_TAG}" - - - name: Remove custom docker config directory - if: always() - run: | - rm -rf .docker-custom diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e447fba47..164eb77f58 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,16 +74,11 @@ We're using the following approach to make it work: For details see [`approved-for-ci-run.yml`](.github/workflows/approved-for-ci-run.yml) -## How do I add the "pinned" tag to an buildtools image? -We use the `pinned` tag for `Dockerfile.buildtools` build images in our CI/CD setup, currently adding the `pinned` tag is a manual operation. +## How do I make build-tools image "pinned" -You can call it from GitHub UI: https://github.com/neondatabase/neon/actions/workflows/update_build_tools_image.yml, -or using GitHub CLI: +It's possible to update the `pinned` tag of the `build-tools` image using the `pin-build-tools-image.yml` workflow. ```bash -gh workflow -R neondatabase/neon run update_build_tools_image.yml \ - -f from-tag=6254913013 \ - -f to-tag=pinned \ - -# Default `-f to-tag` is `pinned`, so the parameter can be omitted. -``` \ No newline at end of file +gh workflow -R neondatabase/neon run pin-build-tools-image.yml \ + -f from-tag=cc98d9b00d670f182c507ae3783342bd7e64c31e +``` diff --git a/Dockerfile.buildtools b/Dockerfile.build-tools similarity index 100% rename from Dockerfile.buildtools rename to Dockerfile.build-tools