From 12e87f0df3f39dfa85c2346695941f4128612866 Mon Sep 17 00:00:00 2001 From: Rory de Zoete <33318916+zoete@users.noreply.github.com> Date: Fri, 19 Aug 2022 12:07:46 +0200 Subject: [PATCH] Update workflow to fix dependency issue (#2309) * Update workflow to fix dependency issue * Update workflow * Update workflow and dockerfile * Specify tag * Update main dockerfile as well * Mirror rust image to docker hub * Update submodule ref Co-authored-by: Rory de Zoete --- .github/workflows/build_and_test.yml | 42 +++++++--------------------- Dockerfile | 7 +++-- Dockerfile.compute-tools | 6 +++- vendor/postgres | 2 +- 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1e71a53f99..3a2e8bad64 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -446,29 +446,13 @@ jobs: - name: Kaniko build compute tools run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:$GITHUB_RUN_ID - promote-image-compute-tools: - runs-on: dev - needs: [ compute-tools-image ] - if: github.event_name != 'workflow_dispatch' - container: amazon/aws-cli - strategy: - fail-fast: false - matrix: - name: [ compute-tools ] - - steps: - - name: Promote image to latest - run: - MANIFEST=$(aws ecr batch-get-image --repository-name ${{ matrix.name }} --image-ids imageTag=$GITHUB_RUN_ID --query 'images[].imageManifest' --output text) && aws ecr put-image --repository-name ${{ matrix.name }} --image-tag latest --image-manifest "$MANIFEST" - compute-node-image: runs-on: dev container: gcr.io/kaniko-project/executor:v1.9.0-debug # note: This image depends on neondatabase/compute-tools:latest (or :thisversion), # which isn't available until after the image is promoted. # Ergo, we must explicitly build and promote compute-tools separately. - needs: - - promote-image-compute-tools + needs: [ compute-tools-image ] steps: - name: Checkout @@ -482,17 +466,17 @@ jobs: - name: Kaniko build compute node working-directory: ./vendor/postgres/ - run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg=COMPUTE_TOOLS_TAG=$GITHUB_RUN_ID --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID + run: /kaniko/executor --snapshotMode=redo --cache=true --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --snapshotMode=redo --context . --build-arg=TAG=$GITHUB_RUN_ID --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:$GITHUB_RUN_ID promote-images: runs-on: dev - needs: [ neon-image, compute-node-image ] + needs: [ neon-image, compute-node-image, compute-tools-image ] if: github.event_name != 'workflow_dispatch' container: amazon/aws-cli strategy: fail-fast: false matrix: - name: [ neon, compute-node ] + name: [ neon, compute-node, compute-tools ] steps: - name: Promote image to latest @@ -509,18 +493,6 @@ jobs: run: | go install github.com/google/go-containerregistry/cmd/crane@31786c6cbb82d6ec4fb8eb79cd9387905130534e # v0.11.0 go install github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@69c85dc22db6511932bbf119e1a0cc5c90c69a7f # v0.6.0 - -# - name: Get build tag -# run: | -# if [[ "$GITHUB_REF_NAME" == "main" ]]; then -# echo "::set-output name=tag::$(git rev-list --count HEAD)" -# elif [[ "$GITHUB_REF_NAME" == "release" ]]; then -# echo "::set-output name=tag::release-$(git rev-list --count HEAD)" -# else -# echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release' " -# echo "::set-output name=tag::$GITHUB_RUN_ID" -# fi -# id: build-tag - name: Configure ECR login run: | @@ -536,6 +508,9 @@ jobs: - name: Pull compute node image from ECR run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node:latest compute-node + - name: Pull rust image from ECR + run: crane pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned rust + - name: Configure docker login run: | # ECR Credential Helper & Docker Hub don't work together in config, hence reset @@ -551,6 +526,9 @@ jobs: - name: Push compute node image to Docker Hub run: crane push compute-node neondatabase/compute-node:${{needs.tag.outputs.build-tag}} + - name: Push rust image to Docker Hub + run: crane push rust neondatabase/rust:pinned + - name: Add latest tag to images if: | (github.ref_name == 'main' || github.ref_name == 'release') && diff --git a/Dockerfile b/Dockerfile index dccf7b6c54..77598fd086 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,12 @@ ### The image itself is mainly used as a container for the binaries and for starting e2e tests with custom parameters. ### By default, the binaries inside the image have some mock parameters and can start, but are not intended to be used ### inside this image in the real deployments. +ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG IMAGE=rust +ARG TAG=pinned # Build Postgres -FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned AS pg-build +FROM $REPOSITORY/$IMAGE:$TAG AS pg-build WORKDIR /home/nonroot COPY vendor/postgres vendor/postgres @@ -17,7 +20,7 @@ RUN set -e \ && tar -C tmp_install -czf /home/nonroot/postgres_install.tar.gz . # Build zenith binaries -FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned AS build +FROM $REPOSITORY/$IMAGE:$TAG AS build WORKDIR /home/nonroot ARG GIT_VERSION=local diff --git a/Dockerfile.compute-tools b/Dockerfile.compute-tools index 05393021c2..47c408bbf2 100644 --- a/Dockerfile.compute-tools +++ b/Dockerfile.compute-tools @@ -1,6 +1,10 @@ # First transient image to build compute_tools binaries # NB: keep in sync with rust image version in .github/workflows/build_and_test.yml -FROM 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned AS rust-build +ARG REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com +ARG IMAGE=rust +ARG TAG=pinned + +FROM $REPOSITORY/$IMAGE:$TAG AS rust-build WORKDIR /home/nonroot # Enable https://github.com/paritytech/cachepot to cache Rust crates' compilation results in Docker builds. diff --git a/vendor/postgres b/vendor/postgres index 7e32bba2aa..3f315a1ec3 160000 --- a/vendor/postgres +++ b/vendor/postgres @@ -1 +1 @@ -Subproject commit 7e32bba2aa2a1752996586bfaf35754f1f0a4d53 +Subproject commit 3f315a1ec336b3a22a09d2015ce91697def4904e