name: Test

on:
  push:
    branches:
    - main
  pull_request:

defaults:
  run:
    shell: bash -ex {0}

concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true

env:
  RUST_BACKTRACE: 1
  COPT: '-Werror'

jobs:
  build-postgres:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]

    env:
      BUILD_TYPE: ${{ matrix.build_type }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 1

      - name: Set pg revision for caching
        id: pg_ver
        run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres)

      - name: Cache postgres build
        id: cache_pg
        uses: actions/cache@v3
        with:
          path: tmp_install/
          key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_ver.outputs.pg_rev }}-${{ hashFiles('Makefile') }}

      - name: Build postgres
        if: steps.cache_pg.outputs.cache-hit != 'true'
        run: mold -run make postgres -j$(nproc)

      # actions/cache@v3 does not allow concurrently using the same cache across job steps, so use a separate cache
      - name: Prepare postgres artifact
        run: tar -C tmp_install/ -czf ./pg.tgz .
      - name: Upload postgres artifact
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          if-no-files-found: error
          name: postgres-${{ runner.os }}-${{ matrix.build_type }}-artifact
          path: ./pg.tgz


  build-neon:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-postgres ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]

    env:
      BUILD_TYPE: ${{ matrix.build_type }}

    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 1

      - name: Get postgres artifact for restoration
        uses: actions/download-artifact@v3
        with:
          name: postgres-${{ runner.os }}-${{ matrix.build_type }}-artifact
          path: ./postgres-artifact/
      - name: Extract postgres artifact
        run: |
          mkdir ./tmp_install/
          tar -xf ./postgres-artifact/pg.tgz -C ./tmp_install/
          rm -rf ./postgres-artifact/

      - name: Cache cargo deps
        id: cache_cargo
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry/
            ~/.cargo/git/
            target/
          # Fall back to older versions of the key, if no cache for current Cargo.lock was found
          key: |
            v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }}
            v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-

      - name: Run cargo build
        run: |
          if [[ $BUILD_TYPE == "debug" ]]; then
            cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
            CARGO_FLAGS=
          elif [[ $BUILD_TYPE == "release" ]]; then
            cov_prefix=()
            CARGO_FLAGS="--release --features profiling"
          fi

          "${cov_prefix[@]}" mold -run cargo build $CARGO_FLAGS --features failpoints --bins --tests

      - name: Run cargo test
        run: |
          if [[ $BUILD_TYPE == "debug" ]]; then
            cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
            CARGO_FLAGS=
          elif [[ $BUILD_TYPE == "release" ]]; then
            cov_prefix=()
            CARGO_FLAGS=--release
          fi

          "${cov_prefix[@]}" cargo test $CARGO_FLAGS

      - name: Install rust binaries
        run: |
          if [[ $BUILD_TYPE == "debug" ]]; then
            cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
          elif [[ $BUILD_TYPE == "release" ]]; then
            cov_prefix=()
          fi

          binaries=$(
            "${cov_prefix[@]}" cargo metadata --format-version=1 --no-deps |
            jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name'
          )

          test_exe_paths=$(
            "${cov_prefix[@]}" cargo test --message-format=json --no-run |
            jq -r '.executable | select(. != null)'
          )

          mkdir -p /tmp/neon/bin/
          mkdir -p /tmp/neon/test_bin/
          mkdir -p /tmp/neon/etc/

          # Keep bloated coverage data files away from the rest of the artifact
          mkdir -p /tmp/coverage/

          # Install target binaries
          for bin in $binaries; do
            SRC=target/$BUILD_TYPE/$bin
            DST=/tmp/neon/bin/$bin
            cp "$SRC" "$DST"
          done

          # Install test executables and write list of all binaries (for code coverage)
          if [[ $BUILD_TYPE == "debug" ]]; then
            for bin in $binaries; do
              echo "/tmp/neon/bin/$bin" >> /tmp/coverage/binaries.list
            done
            for bin in $test_exe_paths; do
              SRC=$bin
              DST=/tmp/neon/test_bin/$(basename $bin)
              cp "$SRC" "$DST"
              echo "$DST" >> /tmp/coverage/binaries.list
            done
          fi

      - name: Install postgres binaries
        run: cp -a tmp_install /tmp/neon/pg_install

      - name: Prepare neon artifact
        run: tar -C /tmp/neon/ -czf ./neon.tgz .

      - name: Upload neon binaries
        uses: actions/upload-artifact@v3
        with:
          retention-days: 7
          if-no-files-found: error
          name: neon-${{ runner.os }}-${{ matrix.build_type }}-${{ matrix.rust_toolchain }}-artifact
          path: ./neon.tgz

      # XXX: keep this after the binaries.list is formed, so the coverage can properly work later
      - name: Merge and upload coverage data
        if: matrix.build_type == 'debug'
        uses: ./.github/actions/save-coverage-data


  pg_regress-tests:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-neon ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 2

      - name: Pytest regress tests
        uses: ./.github/actions/run-python-test-set
        with:
          build_type: ${{ matrix.build_type }}
          rust_toolchain: ${{ matrix.rust_toolchain }}
          test_selection: batch_pg_regress
          needs_postgres_source: true

      - name: Merge and upload coverage data
        if: matrix.build_type == 'debug'
        uses: ./.github/actions/save-coverage-data

  other-tests:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-neon ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug, release ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 2

      - name: Pytest other tests
        uses: ./.github/actions/run-python-test-set
        with:
          build_type: ${{ matrix.build_type }}
          rust_toolchain: ${{ matrix.rust_toolchain }}
          test_selection: batch_others

      - name: Merge and upload coverage data
        if: matrix.build_type == 'debug'
        uses: ./.github/actions/save-coverage-data

  benchmarks:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ build-neon ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ release ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 2

      - name: Pytest benchmarks
        uses: ./.github/actions/run-python-test-set
        with:
          build_type: ${{ matrix.build_type }}
          rust_toolchain: ${{ matrix.rust_toolchain }}
          test_selection: performance
          run_in_parallel: false
          save_perf_report: true
      # XXX: no coverage data handling here, since benchmarks are run on release builds,
      # while coverage is currently collected for the debug ones

  coverage-report:
    runs-on: [ self-hosted, Linux, k8s-runner ]
    needs: [ other-tests, pg_regress-tests ]
    strategy:
      fail-fast: false
      matrix:
        build_type: [ debug ]
        rust_toolchain: [ 1.58 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          submodules: true
          fetch-depth: 1

      - name: Restore cargo deps cache
        id: cache_cargo
        uses: actions/cache@v3
        with:
          path: |
            ~/.cargo/registry/
            ~/.cargo/git/
            target/
          key: v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }}

      - name: Get Neon artifact for restoration
        uses: actions/download-artifact@v3
        with:
          name: neon-${{ runner.os }}-${{ matrix.build_type }}-${{ matrix.rust_toolchain }}-artifact
          path: ./neon-artifact/

      - name: Extract Neon artifact
        run: |
          mkdir -p /tmp/neon/
          tar -xf ./neon-artifact/neon.tgz -C /tmp/neon/
          rm -rf ./neon-artifact/

      - name: Restore coverage data
        uses: actions/download-artifact@v3
        with:
          name: coverage-data-artifact
          path: /tmp/coverage/

      - name: Merge coverage data
        run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge

      - name: Build and upload coverage report
        run: |
          COMMIT_SHA=${{ github.event.pull_request.head.sha }}
          COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}}
          COMMIT_URL=https://github.com/${{ github.repository }}/commit/$COMMIT_SHA

          scripts/coverage \
            --dir=/tmp/coverage report \
            --input-objects=/tmp/coverage/binaries.list \
            --commit-url=$COMMIT_URL \
            --format=github

          REPORT_URL=https://${{ github.repository_owner }}.github.io/zenith-coverage-data/$COMMIT_SHA

          scripts/git-upload \
            --repo=https://${{ secrets.VIP_VAP_ACCESS_TOKEN }}@github.com/${{ github.repository_owner }}/zenith-coverage-data.git \
            --message="Add code coverage for $COMMIT_URL" \
            copy /tmp/coverage/report $COMMIT_SHA # COPY FROM TO_RELATIVE

          # Add link to the coverage report to the commit
          curl -f -X POST \
          https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \
          -H "Accept: application/vnd.github.v3+json" \
          --user "${{ secrets.CI_ACCESS_TOKEN }}" \
          --data \
            "{
              \"state\": \"success\",
              \"context\": \"neon-coverage\",
              \"description\": \"Coverage report is ready\",
              \"target_url\": \"$REPORT_URL\"
            }"

  trigger-e2e-tests:
   runs-on: [ self-hosted, Linux, k8s-runner ]
   needs: [ build-neon ]
   steps:
     - name: Set PR's status to pending and request a remote CI test
       run: |
         COMMIT_SHA=${{ github.event.pull_request.head.sha }}
         COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}}

         REMOTE_REPO="${{ github.repository_owner }}/cloud"

         curl -f -X POST \
         https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \
         -H "Accept: application/vnd.github.v3+json" \
         --user "${{ secrets.CI_ACCESS_TOKEN }}" \
         --data \
           "{
             \"state\": \"pending\",
             \"context\": \"neon-cloud-e2e\",
             \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
           }"

         curl -f -X POST \
         https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \
         -H "Accept: application/vnd.github.v3+json" \
         --user "${{ secrets.CI_ACCESS_TOKEN }}" \
         --data \
           "{
             \"ref\": \"main\",
             \"inputs\": {
               \"ci_job_name\": \"neon-cloud-e2e\",
               \"commit_hash\": \"$COMMIT_SHA\",
               \"remote_repo\": \"${{ github.repository }}\"
             }
           }"