From 2b21d7b5bc7180f606982e401d3a5c4dd130b635 Mon Sep 17 00:00:00 2001 From: Sergey Melnikov Date: Wed, 13 Jul 2022 12:51:20 +0300 Subject: [PATCH] Migrate from CircleCI to Github Actions: docker build and deploy (#1986) --- .circleci/config.yml | 368 ------------------ {.circleci => .github}/ansible/.gitignore | 0 {.circleci => .github}/ansible/ansible.cfg | 0 .../ansible/ansible.ssh.cfg | 0 {.circleci => .github}/ansible/deploy.yaml | 0 .../ansible/get_binaries.sh | 0 .../ansible/neon-stress.hosts | 0 .../ansible/production.hosts | 0 .../ansible/scripts/init_pageserver.sh | 0 .../ansible/scripts/init_safekeeper.sh | 0 {.circleci => .github}/ansible/staging.hosts | 0 .../ansible/systemd/pageserver.service | 0 .../ansible/systemd/safekeeper.service | 0 .../helm-values/neon-stress.proxy-scram.yaml | 0 .../helm-values/neon-stress.proxy.yaml | 0 .../helm-values/production.proxy-scram.yaml | 0 .../helm-values/production.proxy.yaml | 0 .../helm-values/staging.proxy-scram.yaml | 0 .../helm-values/staging.proxy.yaml | 0 .github/workflows/build_and_test.yml | 242 ++++++++++++ 20 files changed, 242 insertions(+), 368 deletions(-) rename {.circleci => .github}/ansible/.gitignore (100%) rename {.circleci => .github}/ansible/ansible.cfg (100%) rename {.circleci => .github}/ansible/ansible.ssh.cfg (100%) rename {.circleci => .github}/ansible/deploy.yaml (100%) rename {.circleci => .github}/ansible/get_binaries.sh (100%) rename {.circleci => .github}/ansible/neon-stress.hosts (100%) rename {.circleci => .github}/ansible/production.hosts (100%) rename {.circleci => .github}/ansible/scripts/init_pageserver.sh (100%) rename {.circleci => .github}/ansible/scripts/init_safekeeper.sh (100%) rename {.circleci => .github}/ansible/staging.hosts (100%) rename {.circleci => .github}/ansible/systemd/pageserver.service (100%) rename {.circleci => .github}/ansible/systemd/safekeeper.service (100%) rename {.circleci => .github}/helm-values/neon-stress.proxy-scram.yaml (100%) rename {.circleci => .github}/helm-values/neon-stress.proxy.yaml (100%) rename {.circleci => .github}/helm-values/production.proxy-scram.yaml (100%) rename {.circleci => .github}/helm-values/production.proxy.yaml (100%) rename {.circleci => .github}/helm-values/staging.proxy-scram.yaml (100%) rename {.circleci => .github}/helm-values/staging.proxy.yaml (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 941849bb0e..00a51eb906 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -325,274 +325,6 @@ jobs: paths: - "*" - # Build neondatabase/neon:latest image and push it to Docker hub - docker-image: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG=$(git log --oneline|wc -l) - docker build \ - --pull \ - --build-arg GIT_VERSION=${CIRCLE_SHA1} \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:latest . - docker push neondatabase/neon:${DOCKER_TAG} - docker push neondatabase/neon:latest - - # Build neondatabase/compute-node:latest image and push it to Docker hub - docker-image-compute: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Build and push compute-tools Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - docker build \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/compute-tools:local \ - --tag neondatabase/compute-tools:latest \ - -f Dockerfile.compute-tools . - # Only push :latest image - docker push neondatabase/compute-tools:latest - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push compute-node Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG=$(git log --oneline|wc -l) - docker build --tag neondatabase/compute-node:${DOCKER_TAG} \ - --tag neondatabase/compute-node:latest vendor/postgres \ - --build-arg COMPUTE_TOOLS_TAG=local - docker push neondatabase/compute-node:${DOCKER_TAG} - docker push neondatabase/compute-node:latest - - # Build production neondatabase/neon:release image and push it to Docker hub - docker-image-release: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG="release-$(git log --oneline|wc -l)" - docker build \ - --pull \ - --build-arg GIT_VERSION=${CIRCLE_SHA1} \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:release . - docker push neondatabase/neon:${DOCKER_TAG} - docker push neondatabase/neon:release - - # Build production neondatabase/compute-node:release image and push it to Docker hub - docker-image-compute-release: - docker: - - image: cimg/base:2021.04 - steps: - - checkout - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Build and push compute-tools Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - docker build \ - --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \ - --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \ - --tag neondatabase/compute-tools:release \ - --tag neondatabase/compute-tools:local \ - -f Dockerfile.compute-tools . - # Only push :release image - docker push neondatabase/compute-tools:release - - run: - name: Init postgres submodule - command: git submodule update --init --depth 1 - - run: - name: Build and push compute-node Docker image - command: | - echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin - DOCKER_TAG="release-$(git log --oneline|wc -l)" - docker build --tag neondatabase/compute-node:${DOCKER_TAG} \ - --tag neondatabase/compute-node:release vendor/postgres \ - --build-arg COMPUTE_TOOLS_TAG=local - docker push neondatabase/compute-node:${DOCKER_TAG} - docker push neondatabase/compute-node:release - - deploy-staging: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - setup_remote_docker - - run: - name: Setup ansible - command: | - pip install --progress-bar off --user ansible boto3 - - run: - name: Redeploy - command: | - cd "$(pwd)/.circleci/ansible" - - ./get_binaries.sh - - echo "${TELEPORT_SSH_KEY}" | tr -d '\n'| base64 --decode >ssh-key - echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub - chmod 0600 ssh-key - ssh-add ssh-key - rm -f ssh-key ssh-key-cert.pub - - ansible-playbook deploy.yaml -i staging.hosts - rm -f neon_install.tar.gz .neon_current_version - - deploy-staging-proxy: - docker: - - image: cimg/base:2021.04 - environment: - KUBECONFIG: .kubeconfig - steps: - - checkout - - run: - name: Store kubeconfig file - command: | - echo "${STAGING_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - run: - name: Setup helm v3 - command: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts - - run: - name: Re-deploy proxy - command: | - DOCKER_TAG=$(git log --oneline|wc -l) - helm upgrade neon-proxy neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s - helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s - - deploy-neon-stress: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - setup_remote_docker - - run: - name: Setup ansible - command: | - pip install --progress-bar off --user ansible boto3 - - run: - name: Redeploy - command: | - cd "$(pwd)/.circleci/ansible" - - ./get_binaries.sh - - echo "${TELEPORT_SSH_KEY}" | tr -d '\n'| base64 --decode >ssh-key - echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub - chmod 0600 ssh-key - ssh-add ssh-key - rm -f ssh-key ssh-key-cert.pub - - ansible-playbook deploy.yaml -i neon-stress.hosts - rm -f neon_install.tar.gz .neon_current_version - - deploy-neon-stress-proxy: - docker: - - image: cimg/base:2021.04 - environment: - KUBECONFIG: .kubeconfig - steps: - - checkout - - run: - name: Store kubeconfig file - command: | - echo "${NEON_STRESS_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - run: - name: Setup helm v3 - command: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts - - run: - name: Re-deploy proxy - command: | - DOCKER_TAG=$(git log --oneline|wc -l) - helm upgrade neon-stress-proxy neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy.yaml --set image.tag=${DOCKER_TAG} --wait - helm upgrade neon-stress-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait - - deploy-release: - docker: - - image: cimg/python:3.10 - steps: - - checkout - - setup_remote_docker - - run: - name: Setup ansible - command: | - pip install --progress-bar off --user ansible boto3 - - run: - name: Redeploy - command: | - cd "$(pwd)/.circleci/ansible" - - RELEASE=true ./get_binaries.sh - - echo "${TELEPORT_SSH_KEY}" | tr -d '\n'| base64 --decode >ssh-key - echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub - chmod 0600 ssh-key - ssh-add ssh-key - rm -f ssh-key ssh-key-cert.pub - - ansible-playbook deploy.yaml -i production.hosts - rm -f neon_install.tar.gz .neon_current_version - - deploy-release-proxy: - docker: - - image: cimg/base:2021.04 - environment: - KUBECONFIG: .kubeconfig - steps: - - checkout - - run: - name: Store kubeconfig file - command: | - echo "${PRODUCTION_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG} - chmod 0600 ${KUBECONFIG} - - run: - name: Setup helm v3 - command: | - curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash - helm repo add neondatabase https://neondatabase.github.io/helm-charts - - run: - name: Re-deploy proxy - command: | - DOCKER_TAG="release-$(git log --oneline|wc -l)" - helm upgrade neon-proxy neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy.yaml --set image.tag=${DOCKER_TAG} --wait - helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait - workflows: build_and_test: jobs: @@ -635,103 +367,3 @@ workflows: save_perf_report: true requires: - build-neon-release - - docker-image: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - main - requires: - - pg_regress-tests-release - - other-tests-release - - docker-image-compute: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - main - requires: - - pg_regress-tests-release - - other-tests-release - - deploy-staging: - # Context gives an ability to login - context: Docker Hub - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - deploy-staging-proxy: - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - - deploy-neon-stress: - # Context gives an ability to login - context: Docker Hub - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - deploy-neon-stress-proxy: - # deploy only for commits to main - filters: - branches: - only: - - main - requires: - - docker-image - - - docker-image-release: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - release - requires: - - pg_regress-tests-release - - other-tests-release - - docker-image-compute-release: - # Context gives an ability to login - context: Docker Hub - # Build image only for commits to main - filters: - branches: - only: - - release - requires: - - pg_regress-tests-release - - other-tests-release - - deploy-release: - # Context gives an ability to login - context: Docker Hub - # deploy only for commits to main - filters: - branches: - only: - - release - requires: - - docker-image-release - - deploy-release-proxy: - # deploy only for commits to main - filters: - branches: - only: - - release - requires: - - docker-image-release diff --git a/.circleci/ansible/.gitignore b/.github/ansible/.gitignore similarity index 100% rename from .circleci/ansible/.gitignore rename to .github/ansible/.gitignore diff --git a/.circleci/ansible/ansible.cfg b/.github/ansible/ansible.cfg similarity index 100% rename from .circleci/ansible/ansible.cfg rename to .github/ansible/ansible.cfg diff --git a/.circleci/ansible/ansible.ssh.cfg b/.github/ansible/ansible.ssh.cfg similarity index 100% rename from .circleci/ansible/ansible.ssh.cfg rename to .github/ansible/ansible.ssh.cfg diff --git a/.circleci/ansible/deploy.yaml b/.github/ansible/deploy.yaml similarity index 100% rename from .circleci/ansible/deploy.yaml rename to .github/ansible/deploy.yaml diff --git a/.circleci/ansible/get_binaries.sh b/.github/ansible/get_binaries.sh similarity index 100% rename from .circleci/ansible/get_binaries.sh rename to .github/ansible/get_binaries.sh diff --git a/.circleci/ansible/neon-stress.hosts b/.github/ansible/neon-stress.hosts similarity index 100% rename from .circleci/ansible/neon-stress.hosts rename to .github/ansible/neon-stress.hosts diff --git a/.circleci/ansible/production.hosts b/.github/ansible/production.hosts similarity index 100% rename from .circleci/ansible/production.hosts rename to .github/ansible/production.hosts diff --git a/.circleci/ansible/scripts/init_pageserver.sh b/.github/ansible/scripts/init_pageserver.sh similarity index 100% rename from .circleci/ansible/scripts/init_pageserver.sh rename to .github/ansible/scripts/init_pageserver.sh diff --git a/.circleci/ansible/scripts/init_safekeeper.sh b/.github/ansible/scripts/init_safekeeper.sh similarity index 100% rename from .circleci/ansible/scripts/init_safekeeper.sh rename to .github/ansible/scripts/init_safekeeper.sh diff --git a/.circleci/ansible/staging.hosts b/.github/ansible/staging.hosts similarity index 100% rename from .circleci/ansible/staging.hosts rename to .github/ansible/staging.hosts diff --git a/.circleci/ansible/systemd/pageserver.service b/.github/ansible/systemd/pageserver.service similarity index 100% rename from .circleci/ansible/systemd/pageserver.service rename to .github/ansible/systemd/pageserver.service diff --git a/.circleci/ansible/systemd/safekeeper.service b/.github/ansible/systemd/safekeeper.service similarity index 100% rename from .circleci/ansible/systemd/safekeeper.service rename to .github/ansible/systemd/safekeeper.service diff --git a/.circleci/helm-values/neon-stress.proxy-scram.yaml b/.github/helm-values/neon-stress.proxy-scram.yaml similarity index 100% rename from .circleci/helm-values/neon-stress.proxy-scram.yaml rename to .github/helm-values/neon-stress.proxy-scram.yaml diff --git a/.circleci/helm-values/neon-stress.proxy.yaml b/.github/helm-values/neon-stress.proxy.yaml similarity index 100% rename from .circleci/helm-values/neon-stress.proxy.yaml rename to .github/helm-values/neon-stress.proxy.yaml diff --git a/.circleci/helm-values/production.proxy-scram.yaml b/.github/helm-values/production.proxy-scram.yaml similarity index 100% rename from .circleci/helm-values/production.proxy-scram.yaml rename to .github/helm-values/production.proxy-scram.yaml diff --git a/.circleci/helm-values/production.proxy.yaml b/.github/helm-values/production.proxy.yaml similarity index 100% rename from .circleci/helm-values/production.proxy.yaml rename to .github/helm-values/production.proxy.yaml diff --git a/.circleci/helm-values/staging.proxy-scram.yaml b/.github/helm-values/staging.proxy-scram.yaml similarity index 100% rename from .circleci/helm-values/staging.proxy-scram.yaml rename to .github/helm-values/staging.proxy-scram.yaml diff --git a/.circleci/helm-values/staging.proxy.yaml b/.github/helm-values/staging.proxy.yaml similarity index 100% rename from .circleci/helm-values/staging.proxy.yaml rename to .github/helm-values/staging.proxy.yaml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 78aa163f3e..857e9e3533 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -390,3 +390,245 @@ jobs: \"remote_repo\": \"${{ github.repository }}\" } }" + + docker-image: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ pg_regress-tests, other-tests ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + outputs: + build-tag: ${{steps.build-tag.outputs.tag}} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver: docker + + - name: Get build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::$(git rev-list --count HEAD)" + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release-$(git rev-list --count HEAD)" + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: build-tag + + - name: Get legacy build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::latest + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: legacy-build-tag + + - name: Build compute-tools Docker image + uses: docker/build-push-action@v2 + with: + context: . + build-args: | + GIT_VERSION="${GITHUB_SHA}" + AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" + pull: true + push: true + tags: neondatabase/neon:${{steps.legacy-build-tag.outputs.tag}}, neondatabase/neon:${{steps.build-tag.outputs.tag}} + + docker-image-compute: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ pg_regress-tests, other-tests ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + outputs: + build-tag: ${{steps.build-tag.outputs.tag}} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver: docker + + - name: Get build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::$(git rev-list --count HEAD)" + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release-$(git rev-list --count HEAD)" + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: build-tag + + - name: Get legacy build tag + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + echo "::set-output name=tag::latest + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + echo "::set-output name=tag::release + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + id: legacy-build-tag + + - name: Build compute-tools Docker image + uses: docker/build-push-action@v2 + with: + context: . + build-args: | + AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" + push: false + file: Dockerfile.compute-tools + tags: neondatabase/compute-tools:local + + - name: Push compute-tools Docker image + uses: docker/build-push-action@v2 + with: + context: . + build-args: | + AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" + AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" + push: true + file: Dockerfile.compute-tools + tags: neondatabase/compute-tools:${{steps.legacy-build-tag.outputs.tag}} + + - name: Build compute-node Docker image + uses: docker/build-push-action@v2 + with: + context: ./vendor/postgres/ + build-args: + COMPUTE_TOOLS_TAG=local + push: true + tags: neondatabase/compute-node:${{steps.legacy-build-tag.outputs.tag}}, neondatabase/compute-node:${{steps.build-tag.outputs.tag}} + + calculate-deploy-targets: + runs-on: [ self-hosted, Linux, k8s-runner ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + outputs: + matrix-include: ${{ steps.set-matrix.outputs.include }} + steps: + - id: set-matrix + run: | + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA"}' + NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA"}' + echo "::set-output name=include::[$STAGING, $NEON_STRESS]" + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA"}' + echo "::set-output name=include::[$PRODUCTION]" + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + + deploy: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ docker-image, calculate-deploy-targets ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + strategy: + matrix: + include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Setup ansible + run: | + pip install --progress-bar off --user ansible boto3 + + - name: Redeploy + run: | + cd "$(pwd)/.github/ansible" + + if [[ "$GITHUB_REF_NAME" == "main" ]]; then + ./get_binaries.sh + elif [[ "$GITHUB_REF_NAME" == "release" ]]; then + RELEASE=true ./get_binaries.sh + else + echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'" + exit 1 + fi + + eval $(ssh-agent) + echo "${{ secrets.TELEPORT_SSH_KEY }}" | tr -d '\n'| base64 --decode >ssh-key + echo "${{ secrets.TELEPORT_SSH_CERT }}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub + chmod 0600 ssh-key + ssh-add ssh-key + rm -f ssh-key ssh-key-cert.pub + + ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts + rm -f neon_install.tar.gz .neon_current_version + + deploy-proxy: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ docker-image, calculate-deploy-targets ] + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + strategy: + matrix: + include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}} + env: + KUBECONFIG: .kubeconfig + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 0 + + - name: Store kubeconfig file + run: | + echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG} + chmod 0600 ${KUBECONFIG} + + - name: Setup helm v3 + run: | + curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash + helm repo add neondatabase https://neondatabase.github.io/helm-charts + + - name: Re-deploy proxy + run: | + DOCKER_TAG=${{needs.docker-image.outputs.build-tag}} + helm upgrade ${{ matrix.proxy_job }} neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s + helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s