From 2dfff6a2a3287083a8057a401fe39ec7898367ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JC=20Gr=C3=BCnhage?= Date: Tue, 18 Mar 2025 12:30:49 +0100 Subject: [PATCH] impr(ci): use ghcr.io as the default container registry (#11210) ## Problem Docker Hub has new rate limits coming up, and to avoid problems coming with those we're switching to GHCR. ## Summary of changes - Push images to GHCR initially and distribute them from there - Use images from GHCR in docker-compose --- .github/scripts/generate_image_maps.py | 4 +- .github/workflows/_build-and-test-locally.yml | 8 +- .github/workflows/_check-codestyle-python.yml | 8 +- .github/workflows/_check-codestyle-rust.yml | 7 +- .../workflows/_push-to-container-registry.yml | 2 +- .github/workflows/build-build-tools-image.yml | 39 ++++- .github/workflows/build_and_test.yml | 144 ++++++++++++------ .github/workflows/cargo-deny.yml | 9 +- .github/workflows/pin-build-tools-image.yml | 8 +- .github/workflows/pre-merge-checks.yml | 11 ++ Dockerfile | 2 +- compute/compute-node.Dockerfile | 2 +- docker-compose/compute_wrapper/Dockerfile | 2 +- docker-compose/docker-compose.yml | 14 +- docker-compose/run-tests.sh | 2 +- 15 files changed, 178 insertions(+), 84 deletions(-) diff --git a/.github/scripts/generate_image_maps.py b/.github/scripts/generate_image_maps.py index f67e07024c..d8f910271b 100644 --- a/.github/scripts/generate_image_maps.py +++ b/.github/scripts/generate_image_maps.py @@ -49,10 +49,10 @@ target_stages = ( for component_name, component_images in components.items(): for stage in target_stages: outputs[f"{component_name}-{stage}"] = { - f"docker.io/neondatabase/{component_image}:{source_tag}": [ + f"ghcr.io/neondatabase/{component_image}:{source_tag}": [ f"{registry}/{component_image}:{tag}" for registry, tag in itertools.product(registries[stage], target_tags) - if not (registry == "docker.io/neondatabase" and tag == source_tag) + if not (registry == "ghcr.io/neondatabase" and tag == source_tag) ] for component_image in component_images } diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index 6a2070424a..db1ea464e6 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -46,8 +46,8 @@ jobs: container: image: ${{ inputs.build-tools-image }} credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} # Raise locked memory limit for tokio-epoll-uring. # On 5.10 LTS kernels < 5.10.162 (and generally mainline kernels < 5.12), # io_uring will account the memory of the CQ and SQ as locked. @@ -322,8 +322,8 @@ jobs: container: image: ${{ inputs.build-tools-image }} credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} # for changed limits, see comments on `options:` earlier in this file options: --init --shm-size=512mb --ulimit memlock=67108864:67108864 strategy: diff --git a/.github/workflows/_check-codestyle-python.yml b/.github/workflows/_check-codestyle-python.yml index 9ae28a1379..868ac15f3c 100644 --- a/.github/workflows/_check-codestyle-python.yml +++ b/.github/workflows/_check-codestyle-python.yml @@ -15,11 +15,15 @@ defaults: jobs: check-codestyle-python: runs-on: [ self-hosted, small ] + + permissions: + packages: read + container: image: ${{ inputs.build-tools-image }} credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --init steps: diff --git a/.github/workflows/_check-codestyle-rust.yml b/.github/workflows/_check-codestyle-rust.yml index c4c76914aa..6d517abe72 100644 --- a/.github/workflows/_check-codestyle-rust.yml +++ b/.github/workflows/_check-codestyle-rust.yml @@ -26,11 +26,14 @@ jobs: arch: ${{ fromJson(inputs.archs) }} runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }} + permissions: + packages: read + container: image: ${{ inputs.build-tools-image }} credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --init steps: diff --git a/.github/workflows/_push-to-container-registry.yml b/.github/workflows/_push-to-container-registry.yml index 2dab665f40..949eeca4b1 100644 --- a/.github/workflows/_push-to-container-registry.yml +++ b/.github/workflows/_push-to-container-registry.yml @@ -89,7 +89,7 @@ jobs: uses: docker/login-action@v3 with: registry: ghcr.io - username: ${{ github.repository_owner }} + username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to Docker Hub diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml index 0a7f0cd7a0..4eae242395 100644 --- a/.github/workflows/build-build-tools-image.yml +++ b/.github/workflows/build-build-tools-image.yml @@ -19,7 +19,7 @@ on: value: ${{ jobs.check-image.outputs.tag }} image: description: "build-tools image" - value: neondatabase/build-tools:${{ jobs.check-image.outputs.tag }} + value: ghcr.io/neondatabase/build-tools:${{ jobs.check-image.outputs.tag }} defaults: run: @@ -49,9 +49,18 @@ jobs: everything: ${{ steps.set-more-variables.outputs.everything }} found: ${{ steps.set-more-variables.outputs.found }} + permissions: + packages: read + steps: - uses: actions/checkout@v4 + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Set variables id: set-variables env: @@ -75,7 +84,7 @@ jobs: contains(fromJson(steps.set-variables.outputs.debians), 'bullseye') && contains(fromJson(steps.set-variables.outputs.debians), 'bookworm') }} run: | - if docker manifest inspect neondatabase/build-tools:${IMAGE_TAG}; then + if docker manifest inspect ghcr.io/neondatabase/build-tools:${IMAGE_TAG}; then found=true else found=false @@ -93,6 +102,9 @@ jobs: arch: ${{ fromJson(needs.check-image.outputs.archs) }} debian: ${{ fromJson(needs.check-image.outputs.debians) }} + permissions: + packages: write + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: @@ -108,6 +120,12 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/login-action@v3 with: registry: cache.neon.build @@ -126,18 +144,27 @@ jobs: cache-from: type=registry,ref=cache.neon.build/build-tools:cache-${{ matrix.debian }}-${{ matrix.arch }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/build-tools:cache-{0}-{1},mode=max', matrix.debian, matrix.arch) || '' }} tags: | - neondatabase/build-tools:${{ needs.check-image.outputs.tag }}-${{ matrix.debian }}-${{ matrix.arch }} + ghcr.io/neondatabase/build-tools:${{ needs.check-image.outputs.tag }}-${{ matrix.debian }}-${{ matrix.arch }} merge-images: needs: [ check-image, build-image ] runs-on: ubuntu-22.04 + permissions: + packages: write + steps: - uses: docker/login-action@v3 with: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Create multi-arch image env: DEFAULT_DEBIAN_VERSION: bookworm @@ -147,14 +174,14 @@ jobs: IMAGE_TAG: ${{ needs.check-image.outputs.tag }} run: | for debian in ${DEBIANS}; do - tags=("-t" "neondatabase/build-tools:${IMAGE_TAG}-${debian}") + tags=("-t" "ghcr.io/neondatabase/build-tools:${IMAGE_TAG}-${debian}") if [ "${EVERYTHING}" == "true" ] && [ "${debian}" == "${DEFAULT_DEBIAN_VERSION}" ]; then - tags+=("-t" "neondatabase/build-tools:${IMAGE_TAG}") + tags+=("-t" "ghcr.io/neondatabase/build-tools:${IMAGE_TAG}") fi for arch in ${ARCHS}; do - tags+=("neondatabase/build-tools:${IMAGE_TAG}-${debian}-${arch}") + tags+=("ghcr.io/neondatabase/build-tools:${IMAGE_TAG}-${debian}-${arch}") done docker buildx imagetools create "${tags[@]}" diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 0980561345..1762cd9644 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -89,8 +89,8 @@ jobs: container: image: ${{ needs.build-build-tools-image.outputs.image }} credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --init steps: @@ -209,8 +209,8 @@ jobs: container: image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --init steps: - name: Checkout @@ -314,8 +314,8 @@ jobs: container: image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --init steps: @@ -367,8 +367,8 @@ jobs: container: image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --init strategy: fail-fast: false @@ -494,6 +494,9 @@ jobs: runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + permissions: + packages: write + steps: - uses: actions/checkout@v4 with: @@ -509,6 +512,12 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/login-action@v3 with: registry: cache.neon.build @@ -533,7 +542,7 @@ jobs: cache-from: type=registry,ref=cache.neon.build/neon:cache-bookworm-${{ matrix.arch }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/neon:cache-{0}-{1},mode=max', 'bookworm', matrix.arch) || '' }} tags: | - neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-${{ matrix.arch }} + ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-${{ matrix.arch }} neon-image: needs: [ neon-image-arch, meta ] @@ -543,19 +552,21 @@ jobs: id-token: write # aws-actions/configure-aws-credentials statuses: write contents: read + packages: write steps: - uses: docker/login-action@v3 with: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Create multi-arch image run: | - docker buildx imagetools create -t neondatabase/neon:${{ needs.meta.outputs.build-tag }} \ - -t neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm \ - neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-x64 \ - neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-arm64 + docker buildx imagetools create -t ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }} \ + -t ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm \ + ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-x64 \ + ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-arm64 compute-node-image-arch: needs: [ check-permissions, build-build-tools-image, meta ] @@ -564,6 +575,7 @@ jobs: id-token: write # aws-actions/configure-aws-credentials statuses: write contents: read + packages: write strategy: fail-fast: false matrix: @@ -604,6 +616,12 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/login-action@v3 with: registry: cache.neon.build @@ -627,7 +645,7 @@ jobs: cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-node-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }} tags: | - neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-${{ matrix.arch }} + ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-${{ matrix.arch }} - name: Build neon extensions test image if: matrix.version.pg >= 'v16' @@ -647,7 +665,7 @@ jobs: target: extension-tests cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} tags: | - neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{needs.meta.outputs.build-tag}}-${{ matrix.version.debian }}-${{ matrix.arch }} + ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{needs.meta.outputs.build-tag}}-${{ matrix.version.debian }}-${{ matrix.arch }} compute-node-image: needs: [ compute-node-image-arch, meta ] @@ -656,6 +674,7 @@ jobs: id-token: write # aws-actions/configure-aws-credentials statuses: write contents: read + packages: write runs-on: ubuntu-22.04 strategy: @@ -674,28 +693,32 @@ jobs: steps: - uses: docker/login-action@v3 with: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Create multi-arch compute-node image run: | - docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ - -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \ - neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ - neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 + docker buildx imagetools create -t ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ + -t ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \ + ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ + ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 - name: Create multi-arch neon-test-extensions image if: matrix.version.pg >= 'v16' run: | - docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ - -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \ - neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ - neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 + docker buildx imagetools create -t ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ + -t ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \ + ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ + ghcr.io/neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 vm-compute-node-image-arch: needs: [ check-permissions, meta, compute-node-image ] if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + permissions: + contents: read + packages: write strategy: fail-fast: false matrix: @@ -723,31 +746,34 @@ jobs: - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193 - uses: docker/login-action@v3 with: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} # Note: we need a separate pull step here because otherwise vm-builder will try to pull, and # it won't have the proper authentication (written at v0.6.0) - name: Pulling compute-node image run: | - docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} + docker pull ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} - name: Build vm image run: | ./vm-builder \ -size=2G \ -spec=compute/vm-image-spec-${{ matrix.version.debian }}.yaml \ - -src=neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ - -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }} \ + -src=ghcr.io/neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ + -dst=ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }} \ -target-arch=linux/${{ matrix.arch }} - name: Pushing vm-compute-node image run: | - docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }} + docker push ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.arch }} vm-compute-node-image: needs: [ vm-compute-node-image-arch, meta ] if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + permissions: + packages: write runs-on: ubuntu-22.04 strategy: matrix: @@ -760,14 +786,15 @@ jobs: steps: - uses: docker/login-action@v3 with: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Create multi-arch compute-node image run: | - docker buildx imagetools create -t neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ - neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-amd64 \ - neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-arm64 + docker buildx imagetools create -t ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \ + ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-amd64 \ + ghcr.io/neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-arm64 test-images: @@ -785,18 +812,28 @@ jobs: arch: [ x64, arm64 ] pg_version: [v16, v17] + permissions: + packages: read + runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }} steps: - uses: actions/checkout@v4 - uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193 + - uses: docker/login-action@v3 with: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library. + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # `ghcr.io/neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library. # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify. # Regular pageserver version string looks like # Neon page server git-env:32d14403bd6ab4f4520a94cbfd81a6acef7a526c failpoints: true, features: [] @@ -807,7 +844,7 @@ jobs: shell: bash # ensure no set -e for better error messages if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} run: | - pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") + pageserver_version=$(docker run --rm ghcr.io/neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") echo "Pageserver version string: $pageserver_version" @@ -978,18 +1015,21 @@ jobs: acr-registry-name: ${{ vars.AZURE_PROD_REGISTRY_NAME }} secrets: inherit - push-neon-test-extensions-image-ghcr: + push-neon-test-extensions-image-dockerhub: if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} needs: [ meta, compute-node-image ] uses: ./.github/workflows/_push-to-container-registry.yml + permissions: + packages: write + id-token: write with: image-map: | { - "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [ - "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}" + "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [ + "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}" ], - "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [ - "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}" + "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [ + "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}" ] } secrets: inherit @@ -998,14 +1038,17 @@ jobs: if: ${{ needs.meta.outputs.run-kind == 'push-main' }} needs: [ meta, compute-node-image ] uses: ./.github/workflows/_push-to-container-registry.yml + permissions: + packages: write + id-token: write with: image-map: | { - "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [ + "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": [ "docker.io/neondatabase/neon-test-extensions-v16:latest", "ghcr.io/neondatabase/neon-test-extensions-v16:latest" ], - "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [ + "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": [ "docker.io/neondatabase/neon-test-extensions-v17:latest", "ghcr.io/neondatabase/neon-test-extensions-v17:latest" ] @@ -1016,14 +1059,17 @@ jobs: if: ${{ needs.meta.outputs.run-kind == 'compute-release' }} needs: [ meta ] uses: ./.github/workflows/_push-to-container-registry.yml + permissions: + packages: write + id-token: write with: image-map: | { - "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.release-pr-run-id }}": [ + "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.release-pr-run-id }}": [ "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}", "ghcr.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}" ], - "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.release-pr-run-id }}": [ + "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.release-pr-run-id }}": [ "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}", "ghcr.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}" ] diff --git a/.github/workflows/cargo-deny.yml b/.github/workflows/cargo-deny.yml index 222f7e9787..a4f476c99a 100644 --- a/.github/workflows/cargo-deny.yml +++ b/.github/workflows/cargo-deny.yml @@ -24,11 +24,14 @@ jobs: runs-on: [self-hosted, small] + permissions: + packages: read + container: - image: ${{ inputs.build-tools-image || 'neondatabase/build-tools:pinned' }} + image: ${{ inputs.build-tools-image || 'ghcr.io/neondatabase/build-tools:pinned' }} credentials: - username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} - password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} options: --init steps: diff --git a/.github/workflows/pin-build-tools-image.yml b/.github/workflows/pin-build-tools-image.yml index d2588ba0bf..ddeefe0128 100644 --- a/.github/workflows/pin-build-tools-image.yml +++ b/.github/workflows/pin-build-tools-image.yml @@ -46,8 +46,8 @@ jobs: FROM_TAG: ${{ inputs.from-tag }} TO_TAG: pinned run: | - docker manifest inspect "docker.io/neondatabase/build-tools:${FROM_TAG}" > "${FROM_TAG}.json" - docker manifest inspect "docker.io/neondatabase/build-tools:${TO_TAG}" > "${TO_TAG}.json" + docker manifest inspect "ghcr.io/neondatabase/build-tools:${FROM_TAG}" > "${FROM_TAG}.json" + docker manifest inspect "ghcr.io/neondatabase/build-tools:${TO_TAG}" > "${TO_TAG}.json" if diff "${FROM_TAG}.json" "${TO_TAG}.json"; then skip=true @@ -71,13 +71,13 @@ jobs: with: image-map: | { - "docker.io/neondatabase/build-tools:${{ inputs.from-tag }}-bullseye": [ + "ghcr.io/neondatabase/build-tools:${{ inputs.from-tag }}-bullseye": [ "docker.io/neondatabase/build-tools:pinned-bullseye", "ghcr.io/neondatabase/build-tools:pinned-bullseye", "${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/build-tools:pinned-bullseye", "${{ vars.AZURE_DEV_REGISTRY_NAME }}.azurecr.io/neondatabase/build-tools:pinned-bullseye" ], - "docker.io/neondatabase/build-tools:${{ inputs.from-tag }}-bookworm": [ + "ghcr.io/neondatabase/build-tools:${{ inputs.from-tag }}-bookworm": [ "docker.io/neondatabase/build-tools:pinned-bookworm", "docker.io/neondatabase/build-tools:pinned", "ghcr.io/neondatabase/build-tools:pinned-bookworm", diff --git a/.github/workflows/pre-merge-checks.yml b/.github/workflows/pre-merge-checks.yml index 3bd81f6538..85b131bb11 100644 --- a/.github/workflows/pre-merge-checks.yml +++ b/.github/workflows/pre-merge-checks.yml @@ -19,6 +19,8 @@ permissions: {} jobs: meta: runs-on: ubuntu-22.04 + permissions: + contents: read outputs: python-changed: ${{ steps.python-src.outputs.any_changed }} rust-changed: ${{ steps.rust-src.outputs.any_changed }} @@ -72,6 +74,9 @@ jobs: || needs.meta.outputs.python-changed == 'true' || needs.meta.outputs.rust-changed == 'true' needs: [ meta ] + permissions: + contents: read + packages: write uses: ./.github/workflows/build-build-tools-image.yml with: # Build only one combination to save time @@ -82,6 +87,9 @@ jobs: check-codestyle-python: if: needs.meta.outputs.python-changed == 'true' needs: [ meta, build-build-tools-image ] + permissions: + contents: read + packages: read uses: ./.github/workflows/_check-codestyle-python.yml with: # `-bookworm-x64` suffix should match the combination in `build-build-tools-image` @@ -91,6 +99,9 @@ jobs: check-codestyle-rust: if: needs.meta.outputs.rust-changed == 'true' needs: [ meta, build-build-tools-image ] + permissions: + contents: read + packages: read uses: ./.github/workflows/_check-codestyle-rust.yml with: # `-bookworm-x64` suffix should match the combination in `build-build-tools-image` diff --git a/Dockerfile b/Dockerfile index 83ad86badb..01540e1925 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ### The image itself is mainly used as a container for the binaries and for starting e2e tests with custom parameters. ### By default, the binaries inside the image have some mock parameters and can start, but are not intended to be used ### inside this image in the real deployments. -ARG REPOSITORY=neondatabase +ARG REPOSITORY=ghcr.io/neondatabase ARG IMAGE=build-tools ARG TAG=pinned ARG DEFAULT_PG_VERSION=17 diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index d5483018b4..bdc73ab174 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -77,7 +77,7 @@ # build_and_test.yml github workflow for how that's done. ARG PG_VERSION -ARG REPOSITORY=neondatabase +ARG REPOSITORY=ghcr.io/neondatabase ARG IMAGE=build-tools ARG TAG=pinned ARG BUILD_TAG diff --git a/docker-compose/compute_wrapper/Dockerfile b/docker-compose/compute_wrapper/Dockerfile index b5f0f47ceb..9ef831a9cd 100644 --- a/docker-compose/compute_wrapper/Dockerfile +++ b/docker-compose/compute_wrapper/Dockerfile @@ -1,4 +1,4 @@ -ARG REPOSITORY=neondatabase +ARG REPOSITORY=ghcr.io/neondatabase ARG COMPUTE_IMAGE=compute-node-v14 ARG TAG=latest diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml index 95d4ff7b2a..493a0a5523 100644 --- a/docker-compose/docker-compose.yml +++ b/docker-compose/docker-compose.yml @@ -29,7 +29,7 @@ services: pageserver: restart: always - image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest} + image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest} environment: - AWS_ACCESS_KEY_ID=minio - AWS_SECRET_ACCESS_KEY=password @@ -45,7 +45,7 @@ services: safekeeper1: restart: always - image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest} + image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest} environment: - SAFEKEEPER_ADVERTISE_URL=safekeeper1:5454 - SAFEKEEPER_ID=1 @@ -75,7 +75,7 @@ services: safekeeper2: restart: always - image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest} + image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest} environment: - SAFEKEEPER_ADVERTISE_URL=safekeeper2:5454 - SAFEKEEPER_ID=2 @@ -105,7 +105,7 @@ services: safekeeper3: restart: always - image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest} + image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest} environment: - SAFEKEEPER_ADVERTISE_URL=safekeeper3:5454 - SAFEKEEPER_ID=3 @@ -135,7 +135,7 @@ services: storage_broker: restart: always - image: ${REPOSITORY:-neondatabase}/neon:${TAG:-latest} + image: ${REPOSITORY:-ghcr.io/neondatabase}/neon:${TAG:-latest} ports: - 50051:50051 command: @@ -147,7 +147,7 @@ services: build: context: ./compute_wrapper/ args: - - REPOSITORY=${REPOSITORY:-neondatabase} + - REPOSITORY=${REPOSITORY:-ghcr.io/neondatabase} - COMPUTE_IMAGE=compute-node-v${PG_VERSION:-16} - TAG=${COMPUTE_TAG:-${TAG:-latest}} - http_proxy=${http_proxy:-} @@ -186,7 +186,7 @@ services: neon-test-extensions: profiles: ["test-extensions"] - image: ${REPOSITORY:-neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-16}:${TEST_EXTENSIONS_TAG:-${TAG:-latest}} + image: ${REPOSITORY:-ghcr.io/neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-16}:${TEST_EXTENSIONS_TAG:-${TAG:-latest}} environment: - PGPASSWORD=cloud_admin entrypoint: diff --git a/docker-compose/run-tests.sh b/docker-compose/run-tests.sh index 72ae61b032..3117950cc0 100644 --- a/docker-compose/run-tests.sh +++ b/docker-compose/run-tests.sh @@ -20,4 +20,4 @@ for d in ${LIST}; do done [ -z "${FAILED}" ] && exit 0 echo "${FAILED}" -exit 1 \ No newline at end of file +exit 1