ci: add push dev-build images to aws ecr (#4618)

* ci: add push dev-build images to aws ecr * chore: use toolchain file generation dev-build image tag * chore: change dev-build version * Update .github/workflows/release-dev-builder-images.yaml Co-authored-by: zyy17 <zyylsxm@gmail.com> --------- Co-authored-by: zyy17 <zyylsxm@gmail.com>
2025-12-22 22:20:02 +00:00 · 2024-08-26 17:36:55 +08:00
parent 3297d5f657
commit 55d65da24d
5 changed files with 102 additions and 30 deletions
--- a/.github/actions/build-linux-artifacts/action.yml
+++ b/.github/actions/build-linux-artifacts/action.yml
@@ -17,6 +17,12 @@ inputs:
    description: Enable dev mode, only build standard greptime
    required: false
    default: "false"
+  image-namespace:
+    description: Image Namespace
+    required: true
+  image-registry:
+    description: Image Registry
+    required: true
  working-dir:
    description: Working directory to build the artifacts
    required: false
@@ -31,8 +37,8 @@ runs:
      run: |
        cd ${{ inputs.working-dir }} && \
        make run-it-in-container BUILD_JOBS=4 \
-        IMAGE_NAMESPACE=i8k6a5e1/greptime \
-        IMAGE_REGISTRY=public.ecr.aws
+        IMAGE_NAMESPACE=${{ inputs.image-namespace }} \
+        IMAGE_REGISTRY=${{ inputs.image-registry }}

    - name: Upload sqlness logs
      if: ${{ failure() && inputs.disable-run-tests == 'false' }} # Only upload logs when the integration tests failed.
@@ -51,8 +57,8 @@ runs:
        artifacts-dir: greptime-linux-${{ inputs.arch }}-pyo3-${{ inputs.version }}
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime        
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}

    - name: Build greptime without pyo3
      if: ${{ inputs.dev-mode == 'false' }}
@@ -64,8 +70,8 @@ runs:
        artifacts-dir: greptime-linux-${{ inputs.arch }}-${{ inputs.version }}
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}

    - name: Clean up the target directory # Clean up the target directory for the centos7 base image, or it will still use the objects of last build.
      shell: bash
@@ -82,8 +88,8 @@ runs:
        artifacts-dir: greptime-linux-${{ inputs.arch }}-centos-${{ inputs.version }}
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}

    - name: Build greptime on android base image
      uses: ./.github/actions/build-greptime-binary
@@ -94,5 +100,5 @@ runs:
        version: ${{ inputs.version }}
        working-dir: ${{ inputs.working-dir }}
        build-android-artifacts: true
-        image-registry: public.ecr.aws
-        image-namespace: i8k6a5e1/greptime
+        image-registry: ${{ inputs.image-registry }}
+        image-namespace: ${{ inputs.image-namespace }}
--- a/.github/workflows/dev-build.yml
+++ b/.github/workflows/dev-build.yml
@@ -177,6 +177,8 @@ jobs:
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          dev-mode: true # Only build the standard greptime binary.
          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
@@ -206,6 +208,8 @@ jobs:
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          dev-mode: true # Only build the standard greptime binary.
          working-dir: ${{ env.CHECKOUT_GREPTIMEDB_PATH }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  release-images-to-dockerhub:
    name: Build and push images to DockerHub
--- a/.github/workflows/nightly-build.yml
+++ b/.github/workflows/nightly-build.yml
@@ -154,6 +154,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
@@ -173,6 +175,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  release-images-to-dockerhub:
    name: Build and push images to DockerHub
--- a/.github/workflows/release-dev-builder-images.yaml
+++ b/.github/workflows/release-dev-builder-images.yaml
@@ -3,10 +3,6 @@ name: Release dev-builder images
 on:
  workflow_dispatch: # Allows you to run this workflow manually.
    inputs:
-      version:
-        description: Version of the dev-builder
-        required: false
-        default: latest
      release_dev_builder_ubuntu_image:
        type: boolean
        description: Release dev-builder-ubuntu image
@@ -28,22 +24,85 @@ jobs:
    name: Release dev builder images
    if: ${{ inputs.release_dev_builder_ubuntu_image || inputs.release_dev_builder_centos_image || inputs.release_dev_builder_android_image }} # Only manually trigger this job.
    runs-on: ubuntu-20.04-16-cores
+    outputs:
+      version: ${{ steps.set-version.outputs.version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

+      - name: Configure build image version
+        id: set-version
+        shell: bash
+        run: |
+          commitShortSHA=`echo ${GITHUB_SHA} | cut -c1-8`
+          buildTime=`date +%Y%m%d%H%M%S`
+          BUILD_VERSION="$commitShortSHA-$buildTime"
+          RUST_TOOLCHAIN_VERSION=$(cat rust_toolchain.toml | grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}')
+          VERSION="${RUST_TOOLCHAIN_VERSION}-${BUILD_VERSION}"
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+          echo "version=VERSION" >> $GITHUB_OUTPUT
+
      - name: Build and push dev builder images
        uses: ./.github/actions/build-dev-builder-images
        with:
-          version: ${{ inputs.version }}
+          version: ${{ env.VERSION }}
          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
          build-dev-builder-ubuntu: ${{ inputs.release_dev_builder_ubuntu_image }}
          build-dev-builder-centos: ${{ inputs.release_dev_builder_centos_image }}
          build-dev-builder-android: ${{ inputs.release_dev_builder_android_image }}

+  release-dev-builder-images-ecr:
+    name: Release dev builder images to AWS ECR
+    runs-on: ubuntu-20.04
+    needs: [
+      release-dev-builder-images
+    ]
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ECR_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_ECR_SECRET_ACCESS_KEY }}
+          aws-region: ${{ vars.ECR_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr-public
+        uses: aws-actions/amazon-ecr-login@v2
+        env:
+          AWS_REGION: ${{ vars.ECR_REGION }}
+        with:
+          registry-type: public
+
+      - name: Push dev-builder-ubuntu image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_ubuntu_image }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }}
+
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-ubuntu:latest
+      - name: Push dev-builder-centos image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_centos_image }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }}
+
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-centos:latest
+      - name: Push dev-builder-android image
+        shell: bash
+        if: ${{ inputs.release_dev_builder_android_image }}
+        run: |
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }}
+
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:latest \
+            docker://${{ vars.ECR_IMAGE_REGISTRY }}/${{ vars.ECR_IMAGE_NAMESPACE }}/dev-builder-android:latest
  release-dev-builder-images-cn: # Note: Be careful issue: https://github.com/containers/skopeo/issues/1874 and we decide to use the latest stable skopeo container.
    name: Release dev builder images to CN region
    runs-on: ubuntu-20.04
@@ -51,35 +110,30 @@ jobs:
      release-dev-builder-images
    ]
    steps:
+      - name: Login to AliCloud Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ vars.ACR_IMAGE_REGISTRY }}
+          username: ${{ secrets.ALICLOUD_USERNAME }}
+          password: ${{ secrets.ALICLOUD_PASSWORD }}
+
      - name: Push dev-builder-ubuntu image
        shell: bash
        if: ${{ inputs.release_dev_builder_ubuntu_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ needs.release-dev-builder-images.outputs.version }} \
            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-ubuntu:${{ inputs.version }}

      - name: Push dev-builder-centos image
        shell: bash
        if: ${{ inputs.release_dev_builder_centos_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ needs.release-dev-builder-images.outputs.version }} \
            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-centos:${{ inputs.version }}

      - name: Push dev-builder-android image
        shell: bash
        if: ${{ inputs.release_dev_builder_android_image }}
-        env:
-          DST_REGISTRY_USERNAME: ${{ secrets.ALICLOUD_USERNAME }}
-          DST_REGISTRY_PASSWORD: ${{ secrets.ALICLOUD_PASSWORD }}
        run: |
-          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }} \
-            --dest-creds "$DST_REGISTRY_USERNAME":"$DST_REGISTRY_PASSWORD" \
+          docker run quay.io/skopeo/stable:latest copy -a docker://docker.io/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ needs.release-dev-builder-images.outputs.version }} \
            docker://${{ vars.ACR_IMAGE_REGISTRY }}/${{ vars.IMAGE_NAMESPACE }}/dev-builder-android:${{ inputs.version }}
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -183,6 +183,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
@@ -202,6 +204,8 @@ jobs:
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
+          image-registry: ${{ vars.ECR_IMAGE_REGISTRY }}
+          image-namespace: ${{ vars.ECR_IMAGE_NAMESPACE }}

  build-macos-artifacts:
    name: Build macOS artifacts