mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-06 04:52:55 +00:00
b77dd66bc4
## Problem Retagging container images and pushing container images taken from one registry to another is very tangled up with artifact building and not separated by component. This makes not building compute for storage releases and vice versa pretty tricky. To enable that, I want to clean up retagging and pushing of container images and then continue on making the pipelines for releases leaner by not building unnecessary things. ## Summary of changes - Add a reusable workflow that can push to ACR, ECR and Docker Hub, while being very flexible in terms of source and target images. This allows for retagging and pushing images between container registries. - Stop pushing images to registries aside of docker hub in the jobs that build the images - Split image pushing into 4 different jobs (not mentioning special cases): - neon-dev - neon-prod - compute-dev - compute-prod ## TODO - Consider also using this for `pin-build-tools-image`, as it's basically another instance of the same thing. ## Known limitations - The ECR part of this workflow supports authenticating to multiple AWS accounts and therefore multiple ECR endpoints, but the ACR part only supports one Azure Account. If someone with more knowledge on Azure can tell me whether an equivalent to https://github.com/aws-actions/amazon-ecr-login?tab=readme-ov-file#login-to-ecr-on-multiple-aws-accounts is easily possible, that'd be great. - The `image_map` input is a bit complex. It expects something along the lines of ``` { "docker.io/neondatabase/compute-node-v14:13196061314": [ "docker.io/neondatabase/compute-node-v14:13196061314", "369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:13196061314", "neoneastus2.azurecr.io/neondatabase/compute-node-v14:13196061314" ], "docker.io/neondatabase/compute-node-v15:13196061314": [ "docker.io/neondatabase/compute-node-v15:13196061314", "369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:13196061314", "neoneastus2.azurecr.io/neondatabase/compute-node-v15:13196061314" ] } ``` to map from source to target image. We have a small python step to generate this map for the 4 main image pushing jobs. The concrete example is taken from https://github.com/neondatabase/neon/actions/runs/13196061314/job/36838584098?pr=10613#step:3:6 and shortened to two images.
102 lines
3.5 KiB
YAML
102 lines
3.5 KiB
YAML
name: Push images to Container Registry
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
# Example: {"docker.io/neondatabase/neon:13196061314":["369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:13196061314","neoneastus2.azurecr.io/neondatabase/neon:13196061314"]}
|
|
image-map:
|
|
description: JSON map of images, mapping from a source image to an array of target images that should be pushed.
|
|
required: true
|
|
type: string
|
|
aws-region:
|
|
description: AWS region to log in to. Required when pushing to ECR.
|
|
required: false
|
|
type: string
|
|
aws-account-ids:
|
|
description: Comma separated AWS account IDs to log in to for pushing to ECR. Required when pushing to ECR.
|
|
required: false
|
|
type: string
|
|
azure-client-id:
|
|
description: Client ID of Azure managed identity or Entra app. Required when pushing to ACR.
|
|
required: false
|
|
type: string
|
|
azure-subscription-id:
|
|
description: Azure subscription ID. Required when pushing to ACR.
|
|
required: false
|
|
type: string
|
|
azure-tenant-id:
|
|
description: Azure tenant ID. Required when pushing to ACR.
|
|
required: false
|
|
type: string
|
|
acr-registry-name:
|
|
description: ACR registry name. Required when pushing to ACR.
|
|
required: false
|
|
type: string
|
|
secrets:
|
|
docker-hub-username:
|
|
description: Docker Hub username. Required when pushing to Docker Hub.
|
|
required: false
|
|
docker-hub-password:
|
|
description: Docker Hub password. Required when pushing to Docker Hub.
|
|
required: false
|
|
aws-role-to-assume:
|
|
description: AWS role to assume. Required when pushing to ECR.
|
|
required: false
|
|
|
|
permissions: {}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash -euo pipefail {0}
|
|
|
|
jobs:
|
|
push-to-container-registry:
|
|
runs-on: ubuntu-22.04
|
|
permissions:
|
|
id-token: write # Required for aws/azure login
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
with:
|
|
sparse-checkout: scripts/push_with_image_map.py
|
|
sparse-checkout-cone-mode: false
|
|
|
|
- name: Print image-map
|
|
run: echo '${{ inputs.image-map }}' | jq
|
|
|
|
- name: Configure AWS credentials
|
|
if: contains(inputs.image-map, 'amazonaws.com/')
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: "${{ inputs.aws-region }}"
|
|
role-to-assume: "${{ secrets.aws-role-to-assume }}"
|
|
role-duration-seconds: 3600
|
|
|
|
- name: Login to ECR
|
|
if: contains(inputs.image-map, 'amazonaws.com/')
|
|
uses: aws-actions/amazon-ecr-login@v2
|
|
with:
|
|
registries: "${{ inputs.aws-account-ids }}"
|
|
|
|
- name: Configure Azure credentials
|
|
if: contains(inputs.image-map, 'azurecr.io/')
|
|
uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # @v2.1.1
|
|
with:
|
|
client-id: ${{ inputs.azure-client-id }}
|
|
subscription-id: ${{ inputs.azure-subscription-id }}
|
|
tenant-id: ${{ inputs.azure-tenant-id }}
|
|
|
|
- name: Login to ACR
|
|
if: contains(inputs.image-map, 'azurecr.io/')
|
|
run: |
|
|
az acr login --name=${{ inputs.acr-registry-name }}
|
|
|
|
- name: Log in to Docker Hub
|
|
uses: docker/login-action@v3
|
|
with:
|
|
username: ${{ secrets.docker-hub-username }}
|
|
password: ${{ secrets.docker-hub-password }}
|
|
|
|
- name: Copy docker images to target registries
|
|
run: python scripts/push_with_image_map.py
|
|
env:
|
|
IMAGE_MAP: ${{ inputs.image-map }}
|