neon/.github/workflows/build_and_test.yml

name: Test

on:
  push:
    branches:
    - main
  pull_request:

defaults:
  run:
    shell: bash -ex {0}

concurrency:
  # Allow only one workflow per any non-`main` branch.
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }}
  cancel-in-progress: true

env:
  RUST_BACKTRACE: 1
  COPT: '-Werror'

jobs:
  build-postgres:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]

    env:
      BUILD_TYPE: ${{ matrix.build_type }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 1

      - name: Set pg revision for caching
        id: pg_ver
        run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres)

      - name: Cache postgres build
        id: cache_pg
        uses: actions/cache@v3
        with:
          path: tmp_install/
          key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_ver.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

      - name: Build postgres
        if: steps.cache_pg.outputs.cache-hit != 'true'
        run: mold -run make postgres -j$(nproc)

      # actions/cache@v3 does not allow concurrently using the same cache across job steps, so use a separate cache
      - name: Prepare postgres artifact
        run: tar -C tmp_install/ -czf ./pg.tgz .
      - name: Upload postgres artifact
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          if-no-files-found: error
          name: postgres-${{ runner.os }}-${{ matrix.build_type }}-artifact
          path: ./pg.tgz


  build-neon:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-postgres ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]

    env:
      BUILD_TYPE: ${{ matrix.build_type }}

    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 1

      - name: Get postgres artifact for restoration
        uses: actions/download-artifact@v3
        with:
          name: postgres-${{ runner.os }}-${{ matrix.build_type }}-artifact
          path: ./postgres-artifact/
      - name: Extract postgres artifact
        run: |
          mkdir ./tmp_install/
          tar -xf ./postgres-artifact/pg.tgz -C ./tmp_install/
          rm -rf ./postgres-artifact/

      - name: Cache cargo deps
        id: cache_cargo
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry/
            ~/.cargo/git/
            target/
          # Fall back to older versions of the key, if no cache for current Cargo.lock was found
          key: |
            v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }}
            v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-

      - name: Run cargo build
        run: |
          if [[ $BUILD_TYPE == "debug" ]]; then
            cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
            CARGO_FLAGS=
          elif [[ $BUILD_TYPE == "release" ]]; then
            cov_prefix=()
            CARGO_FLAGS="--release --features profiling"
          fi

          "${cov_prefix[@]}" mold -run cargo build $CARGO_FLAGS --features failpoints --bins --tests

      - name: Run cargo test
        run: |
          if [[ $BUILD_TYPE == "debug" ]]; then
            cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
            CARGO_FLAGS=
          elif [[ $BUILD_TYPE == "release" ]]; then
            cov_prefix=()
            CARGO_FLAGS=--release
          fi

          "${cov_prefix[@]}" cargo test $CARGO_FLAGS

      - name: Install rust binaries
        run: |
          if [[ $BUILD_TYPE == "debug" ]]; then
            cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
          elif [[ $BUILD_TYPE == "release" ]]; then
            cov_prefix=()
          fi

          binaries=$(
            "${cov_prefix[@]}" cargo metadata --format-version=1 --no-deps |
            jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name'
          )

          test_exe_paths=$(
            "${cov_prefix[@]}" cargo test --message-format=json --no-run |
            jq -r '.executable | select(. != null)'
          )

          mkdir -p /tmp/neon/bin/
          mkdir -p /tmp/neon/test_bin/
          mkdir -p /tmp/neon/etc/

          # Keep bloated coverage data files away from the rest of the artifact
          mkdir -p /tmp/coverage/

          # Install target binaries
          for bin in $binaries; do
            SRC=target/$BUILD_TYPE/$bin
            DST=/tmp/neon/bin/$bin
            cp "$SRC" "$DST"
          done

          # Install test executables and write list of all binaries (for code coverage)
          if [[ $BUILD_TYPE == "debug" ]]; then
            for bin in $binaries; do
              echo "/tmp/neon/bin/$bin" >> /tmp/coverage/binaries.list
            done
            for bin in $test_exe_paths; do
              SRC=$bin
              DST=/tmp/neon/test_bin/$(basename $bin)

              # We don't need debug symbols for code coverage, so strip them out to make
              # the artifact smaller.
              strip "$SRC" -o "$DST"
              echo "$DST" >> /tmp/coverage/binaries.list
            done
          fi

      - name: Install postgres binaries
        run: cp -a tmp_install /tmp/neon/pg_install

      - name: Prepare neon artifact
        run: tar -C /tmp/neon/ -czf ./neon.tgz .

      - name: Upload neon binaries
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          if-no-files-found: error
          name: neon-${{ runner.os }}-${{ matrix.build_type }}-${{ matrix.rust_toolchain }}-artifact
          path: ./neon.tgz

      # XXX: keep this after the binaries.list is formed, so the coverage can properly work later
      - name: Merge and upload coverage data
        if: matrix.build_type == 'debug'
        uses: ./.github/actions/save-coverage-data


  pg_regress-tests:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-neon ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 2

      - name: Pytest regress tests
        uses: ./.github/actions/run-python-test-set
        with:
          build_type: ${{ matrix.build_type }}
          rust_toolchain: ${{ matrix.rust_toolchain }}
          test_selection: batch_pg_regress
          needs_postgres_source: true

      - name: Merge and upload coverage data
        if: matrix.build_type == 'debug'
        uses: ./.github/actions/save-coverage-data

  other-tests:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-neon ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 2

      - name: Pytest other tests
        uses: ./.github/actions/run-python-test-set
        with:
          build_type: ${{ matrix.build_type }}
          rust_toolchain: ${{ matrix.rust_toolchain }}
          test_selection: batch_others

      - name: Merge and upload coverage data
        if: matrix.build_type == 'debug'
        uses: ./.github/actions/save-coverage-data

  benchmarks:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-neon ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ release ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 2

      - name: Pytest benchmarks
        uses: ./.github/actions/run-python-test-set
        with:
          build_type: ${{ matrix.build_type }}
          rust_toolchain: ${{ matrix.rust_toolchain }}
          test_selection: performance
          run_in_parallel: false
          save_perf_report: true
        env:
          VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
          PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
      # XXX: no coverage data handling here, since benchmarks are run on release builds,
      # while coverage is currently collected for the debug ones

  coverage-report:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ other-tests, pg_regress-tests ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 1

      - name: Restore cargo deps cache
        id: cache_cargo
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry/
            ~/.cargo/git/
            target/
          key: v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }}

      - name: Get Neon artifact for restoration
        uses: actions/download-artifact@v3
        with:
          name: neon-${{ runner.os }}-${{ matrix.build_type }}-${{ matrix.rust_toolchain }}-artifact
          path: ./neon-artifact/

      - name: Extract Neon artifact
        run: |
          mkdir -p /tmp/neon/
          tar -xf ./neon-artifact/neon.tgz -C /tmp/neon/
          rm -rf ./neon-artifact/

      - name: Restore coverage data
        uses: actions/download-artifact@v3
        with:
          name: coverage-data-artifact
          path: /tmp/coverage/

      - name: Merge coverage data
        run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge

      - name: Build and upload coverage report
        run: |
          COMMIT_SHA=${{ github.event.pull_request.head.sha }}
          COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}}
          COMMIT_URL=https://github.com/${{ github.repository }}/commit/$COMMIT_SHA

          scripts/coverage \
            --dir=/tmp/coverage report \
            --input-objects=/tmp/coverage/binaries.list \
            --commit-url=$COMMIT_URL \
            --format=github

          REPORT_URL=https://${{ github.repository_owner }}.github.io/zenith-coverage-data/$COMMIT_SHA

          scripts/git-upload \
            --repo=https://${{ secrets.VIP_VAP_ACCESS_TOKEN }}@github.com/${{ github.repository_owner }}/zenith-coverage-data.git \
            --message="Add code coverage for $COMMIT_URL" \
            copy /tmp/coverage/report $COMMIT_SHA # COPY FROM TO_RELATIVE

          # Add link to the coverage report to the commit
          curl -f -X POST \
          https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \
          -H "Accept: application/vnd.github.v3+json" \
          --user "${{ secrets.CI_ACCESS_TOKEN }}" \
          --data \
            "{
              \"state\": \"success\",
              \"context\": \"neon-coverage\",
              \"description\": \"Coverage report is ready\",
              \"target_url\": \"$REPORT_URL\"
            }"

  trigger-e2e-tests:
   runs-on: [ self-hosted, Linux, k8s-runner ]
   needs: [ build-neon ]
   steps:
     - name: Set PR's status to pending and request a remote CI test
       run: |
         COMMIT_SHA=${{ github.event.pull_request.head.sha }}
         COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}}

         REMOTE_REPO="${{ github.repository_owner }}/cloud"

         curl -f -X POST \
         https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \
         -H "Accept: application/vnd.github.v3+json" \
         --user "${{ secrets.CI_ACCESS_TOKEN }}" \
         --data \
           "{
             \"state\": \"pending\",
             \"context\": \"neon-cloud-e2e\",
             \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
           }"

         curl -f -X POST \
         https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \
         -H "Accept: application/vnd.github.v3+json" \
         --user "${{ secrets.CI_ACCESS_TOKEN }}" \
         --data \
           "{
             \"ref\": \"main\",
             \"inputs\": {
               \"ci_job_name\": \"neon-cloud-e2e\",
               \"commit_hash\": \"$COMMIT_SHA\",
               \"remote_repo\": \"${{ github.repository }}\"
             }
           }"

  docker-image:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ pg_regress-tests, other-tests ]
    if: |
      (github.ref_name == 'main' || github.ref_name == 'release') &&
      github.event_name != 'workflow_dispatch'
    outputs:
      build-tag: ${{steps.build-tag.outputs.tag}}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 0

      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
        with:
          driver: docker

      - name: Get build tag
        run: |
          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
            echo "::set-output name=tag::$(git rev-list --count HEAD)"
          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
            echo "::set-output name=tag::release-$(git rev-list --count HEAD)"
          else
            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
            exit 1
          fi
        id: build-tag

      - name: Get legacy build tag
        run: |
          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
            echo "::set-output name=tag::latest
          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
            echo "::set-output name=tag::release
          else
            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
            exit 1
          fi
        id: legacy-build-tag

      - name: Build neon Docker image
        uses: docker/build-push-action@v2
        with:
          context: .
          build-args: |
            GIT_VERSION="${{github.sha}}"
            AWS_ACCESS_KEY_ID="${{secrets.CACHEPOT_AWS_ACCESS_KEY_ID}}"
            AWS_SECRET_ACCESS_KEY="${{secrets.CACHEPOT_AWS_SECRET_ACCESS_KEY}}"
          pull: true
          push: true
          tags: neondatabase/neon:${{steps.legacy-build-tag.outputs.tag}}, neondatabase/neon:${{steps.build-tag.outputs.tag}}

  docker-image-compute:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ pg_regress-tests, other-tests ]
    if: |
      (github.ref_name == 'main' || github.ref_name == 'release') &&
      github.event_name != 'workflow_dispatch'
    outputs:
      build-tag: ${{steps.build-tag.outputs.tag}}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 0

      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
        with:
          driver: docker

      - name: Get build tag
        run: |
          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
            echo "::set-output name=tag::$(git rev-list --count HEAD)"
          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
            echo "::set-output name=tag::release-$(git rev-list --count HEAD)"
          else
            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
            exit 1
          fi
        id: build-tag

      - name: Get legacy build tag
        run: |
          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
            echo "::set-output name=tag::latest
          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
            echo "::set-output name=tag::release
          else
            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
            exit 1
          fi
        id: legacy-build-tag

      - name: Build compute-tools Docker image
        uses: docker/build-push-action@v2
        with:
          context: .
          build-args: |
            GIT_VERSION="${{github.sha}}"
            AWS_ACCESS_KEY_ID="${{secrets.CACHEPOT_AWS_ACCESS_KEY_ID}}"
            AWS_SECRET_ACCESS_KEY="${{secrets.CACHEPOT_AWS_SECRET_ACCESS_KEY}}"
          push: false
          file: Dockerfile.compute-tools
          tags: neondatabase/compute-tools:local

      - name: Push compute-tools Docker image
        uses: docker/build-push-action@v2
        with:
          context: .
          build-args: |
            GIT_VERSION="${{github.sha}}"
            AWS_ACCESS_KEY_ID="${{secrets.CACHEPOT_AWS_ACCESS_KEY_ID}}"
            AWS_SECRET_ACCESS_KEY="${{secrets.CACHEPOT_AWS_SECRET_ACCESS_KEY}}"
          push: true
          file: Dockerfile.compute-tools
          tags: neondatabase/compute-tools:${{steps.legacy-build-tag.outputs.tag}}

      - name: Build compute-node Docker image
        uses: docker/build-push-action@v2
        with:
          context: ./vendor/postgres/
          build-args:
            COMPUTE_TOOLS_TAG=local
          push: true
          tags: neondatabase/compute-node:${{steps.legacy-build-tag.outputs.tag}}, neondatabase/compute-node:${{steps.build-tag.outputs.tag}}

  calculate-deploy-targets:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    if: |
      (github.ref_name == 'main' || github.ref_name == 'release') &&
      github.event_name != 'workflow_dispatch'
    outputs:
      matrix-include: ${{ steps.set-matrix.outputs.include }}
    steps:
      - id: set-matrix
        run: |
          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
            STAGING='{"env_name": "staging", "proxy_job": "neon-proxy", "proxy_config": "staging.proxy", "kubeconfig_secret": "STAGING_KUBECONFIG_DATA"}'
            NEON_STRESS='{"env_name": "neon-stress", "proxy_job": "neon-stress-proxy", "proxy_config": "neon-stress.proxy", "kubeconfig_secret": "NEON_STRESS_KUBECONFIG_DATA"}'
            echo "::set-output name=include::[$STAGING, $NEON_STRESS]"
          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
            PRODUCTION='{"env_name": "production", "proxy_job": "neon-proxy", "proxy_config": "production.proxy", "kubeconfig_secret": "PRODUCTION_KUBECONFIG_DATA"}'
            echo "::set-output name=include::[$PRODUCTION]"
          else
            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
            exit 1
          fi

  deploy:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    # We need both storage **and** compute images for deploy, because control plane
    # picks the compute version based on the storage version. If it notices a fresh
    # storage it may bump the compute version. And if compute image failed to build
    # it may break things badly.
    needs: [ docker-image, docker-image-compute, calculate-deploy-targets ]
    if: |
      (github.ref_name == 'main' || github.ref_name == 'release') &&
      github.event_name != 'workflow_dispatch'
    strategy:
      matrix:
        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 0

      - name: Setup ansible
        run: |
          pip install --progress-bar off --user ansible boto3

      - name: Redeploy
        run: |
          cd "$(pwd)/.github/ansible"

          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
            ./get_binaries.sh
          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
            RELEASE=true ./get_binaries.sh
          else
            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
            exit 1
          fi

          eval $(ssh-agent)
          echo "${{ secrets.TELEPORT_SSH_KEY }}"  | tr -d '\n'| base64 --decode >ssh-key
          echo "${{ secrets.TELEPORT_SSH_CERT }}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
          chmod 0600 ssh-key
          ssh-add ssh-key
          rm -f ssh-key ssh-key-cert.pub

          ansible-playbook deploy.yaml -i ${{ matrix.env_name }}.hosts
          rm -f neon_install.tar.gz .neon_current_version

  deploy-proxy:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    # Compute image isn't strictly required for proxy deploy, but let's still wait for it
    # to run all deploy jobs consistently.
    needs: [ docker-image, docker-image-compute, calculate-deploy-targets ]
    if: |
      (github.ref_name == 'main' || github.ref_name == 'release') &&
      github.event_name != 'workflow_dispatch'
    strategy:
      matrix:
        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
    env:
      KUBECONFIG: .kubeconfig
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 0

      - name: Store kubeconfig file
        run: |
          echo "${{ secrets[matrix.kubeconfig_secret] }}" | base64 --decode > ${KUBECONFIG}
          chmod 0600 ${KUBECONFIG}

      - name: Setup helm v3
        run: |
          curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
          helm repo add neondatabase https://neondatabase.github.io/helm-charts

      - name: Re-deploy proxy
        run: |
          DOCKER_TAG=${{needs.docker-image.outputs.build-tag}}
          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s
          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace default --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --wait --timeout 15m0s