neon/.circleci/config.yml

version: 2.1

orbs:
  python: circleci/python@1.4.0

executors:
  zenith-build-executor:
    resource_class: xlarge
    docker:
      - image: cimg/rust:1.55.0

jobs:
  check-codestyle:
    executor: zenith-build-executor
    steps:
      - checkout

      - run:
          name: rustfmt
          when: always
          command: |
            cargo fmt --all -- --check

  # A job to build postgres
  build-postgres:
    executor: zenith-build-executor
    parameters:
      build_type:
        type: enum
        enum: ["debug", "release"]
    environment:
      BUILD_TYPE: << parameters.build_type >>
    steps:
        # Checkout the git repo (circleci doesn't have a flag to enable submodules here)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres

      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}

        # FIXME We could cache our own docker container, instead of installing packages every time.
      - run:
          name: apt install dependencies
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              sudo apt update
              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
            fi

        # Build postgres if the restore_cache didn't find a build.
        # `make` can't figure out whether the cache is valid, since
        # it only compares file timestamps.
      - run:
          name: build postgres
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              # "depth 1" saves some time by not cloning the whole repo
              git submodule update --init --depth 1
              make postgres -j8
            fi

      - save_cache:
          name: Save postgres cache
          key: v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}
          paths:
            - tmp_install

  # A job to build zenith rust code
  build-zenith:
    executor: zenith-build-executor
    parameters:
      build_type:
        type: enum
        enum: ["debug", "release"]
    steps:
      - run:
          name: apt install dependencies
          command: |
            sudo apt update
            sudo apt install libssl-dev clang

        # Checkout the git repo (without submodules)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres

      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}

      - restore_cache:
          name: Restore rust cache
          keys:
            # Require an exact match. While an out of date cache might speed up the build,
            # there's no way to clean out old packages, so the cache grows every time something
            # changes.
            - v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}

        # Build the rust code, including test binaries
      - run:
          name: Rust build << parameters.build_type >>
          command: |
            export CARGO_INCREMENTAL=0
            BUILD_TYPE="<< parameters.build_type >>"
            if [[ $BUILD_TYPE == "debug" ]]; then
              echo "Build in debug mode"
              cargo build --bins --tests
            elif [[ $BUILD_TYPE == "release" ]]; then
              echo "Build in release mode"
              cargo build --release --bins --tests
            fi

      - save_cache:
          name: Save rust cache
          key: v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}
          paths:
            - ~/.cargo/registry
            - ~/.cargo/git
            - target

        # Run style checks
        # has to run separately from cargo fmt section
        # since needs to run with dependencies
      - run:
          name: clippy
          command: |
            ./run_clippy.sh

        # Run rust unit tests
      - run: cargo test

        # Install the rust binaries, for use by test jobs
        # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
        # FIXME: this is a really silly way to install; maybe we should just output
        # a tarball as an artifact? Or a .deb package?
      - run:
          name: cargo install
          command: |
            export CARGO_INCREMENTAL=0
            BUILD_TYPE="<< parameters.build_type >>"
            if [[ $BUILD_TYPE == "debug" ]]; then
              echo "Install debug mode"
              CARGO_FLAGS="--debug"
            elif [[ $BUILD_TYPE == "release" ]]; then
              echo "Install release mode"
              # The default is release mode; there is no --release flag.
              CARGO_FLAGS=""
            fi
            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path pageserver
            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path walkeeper
            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path zenith

        # Install the postgres binaries, for use by test jobs
        # FIXME: this is a silly way to do "install"; maybe just output a standard
        # postgres package, whatever the favored form is (tarball? .deb package?)
        # Note that pg_regress needs some build artifacts that probably aren't
        # in the usual package...?
      - run:
          name: postgres install
          command: |
            cp -a tmp_install /tmp/zenith/pg_install

        # Save the rust output binaries for other jobs in this workflow.
      - persist_to_workspace:
          root: /tmp/zenith
          paths:
            - "*"

  check-python:
    executor: python/default
    steps:
      - checkout
      - run:
          name: Install pipenv & deps
          working_directory: test_runner
          command: |
            pip install pipenv
            pipenv install --dev
      - run:
          name: Run yapf to ensure code format
          when: always
          working_directory: test_runner
          command: pipenv run yapf --recursive --diff .
      - run:
          name: Run mypy to check types
          when: always
          working_directory: test_runner
          command: pipenv run mypy .

  run-pytest:
    #description: "Run pytest"
    executor: python/default
    parameters:
      # pytest args to specify the tests to run.
      #
      # This can be a test file name, e.g. 'test_pgbench.py, or a subdirectory,
      # or '-k foobar' to run tests containing string 'foobar'. See pytest man page
      # section SPECIFYING TESTS / SELECTING TESTS for details.
      #
      # Select the type of Rust build. Must be "release" or "debug".
      build_type:
        type: string
        default: "debug"
      # This parameter is required, to prevent the mistake of running all tests in one job.
      test_selection:
        type: string
        default: ""
      # Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr
      extra_params:
        type: string
        default: ""
      needs_postgres_source:
        type: boolean
        default: false
      run_in_parallel:
        type: boolean
        default: true
    steps:
      - attach_workspace:
          at: /tmp/zenith
      - checkout
      - when:
          condition: << parameters.needs_postgres_source >>
          steps:
            - run: git submodule update --init --depth 1
      - run:
          name: Install pipenv & deps
          working_directory: test_runner
          command: |
            pip install pipenv
            pipenv install
      - run:
          name: Run pytest
          working_directory: test_runner
          environment:
            - ZENITH_BIN: /tmp/zenith/bin
            - POSTGRES_DISTRIB_DIR: /tmp/zenith/pg_install
            - TEST_OUTPUT: /tmp/test_output
          command: |
            TEST_SELECTION="<< parameters.test_selection >>"
            EXTRA_PARAMS="<< parameters.extra_params >>"
            if [ -z "$TEST_SELECTION" ]; then
              echo "test_selection must be set"
              exit 1
            fi
            if << parameters.run_in_parallel >>; then
              EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
            fi;
            # Run the tests.
            #
            # The junit.xml file allows CircleCI to display more fine-grained test information
            # in its "Tests" tab in the results page.
            # --verbose prints name of each test (helpful when there are
            # multiple tests in one file)
            # -rA prints summary in the end
            # -n4 uses four processes to run tests via pytest-xdist
            # -s is not used to prevent pytest from capturing output, because tests are running
            # in parallel and logs are mixed between different tests
            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
      - run:
          # CircleCI artifacts are preserved one file at a time, so skipping
          # this step isn't a good idea. If you want to extract the
          # pageserver state, perhaps a tarball would be a better idea.
          name: Delete all data but logs
          when: always
          command: |
            du -sh /tmp/test_output/*
            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "safekeeper.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" -delete
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
      # The store_test_results step tells CircleCI where to find the junit.xml file.
      - store_test_results:
          path: /tmp/test_output

  # Build zenithdb/zenith:latest image and push it to Docker hub
  docker-image:
    docker:
      - image: cimg/base:2021.04
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run:
          name: Init postgres submodule
          command: git submodule update --init --depth 1
      - run:
          name: Build and push Docker image
          command: |
            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest

  # Trigger a new remote CI job
  remote-ci-trigger:
    docker:
      - image: cimg/base:2021.04
    parameters:
      remote_repo:
        type: string
    environment:
      REMOTE_REPO: << parameters.remote_repo >>
    steps:
      - run:
          name: Set PR's status to pending
          command: |
            LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME

            curl -f -X POST \
            https://api.github.com/repos/$LOCAL_REPO/statuses/$CIRCLE_SHA1 \
            -H "Accept: application/vnd.github.v3+json" \
            --user "$CI_ACCESS_TOKEN" \
            --data \
              "{
                \"state\": \"pending\",
                \"context\": \"zenith-remote-ci\",
                \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\"
              }"
      - run:
          name: Request a remote CI test
          command: |
            LOCAL_REPO=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME

            curl -f -X POST \
            https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \
            -H "Accept: application/vnd.github.v3+json" \
            --user "$CI_ACCESS_TOKEN" \
            --data \
              "{
                \"ref\": \"main\",
                \"inputs\": {
                  \"ci_job_name\": \"zenith-remote-ci\",
                  \"commit_hash\": \"$CIRCLE_SHA1\",
                  \"remote_repo\": \"$LOCAL_REPO\"
                }
              }"

workflows:
  build_and_test:
    jobs:
      - check-codestyle
      - check-python
      - build-postgres:
          name: build-postgres-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
      - build-zenith:
          name: build-zenith-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          requires:
            - build-postgres-<< matrix.build_type >>
      - run-pytest:
          name: pg_regress-tests-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_pg_regress
          needs_postgres_source: true
          requires:
            - build-zenith-<< matrix.build_type >>
      - run-pytest:
          name: other-tests-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_others
          requires:
            - build-zenith-<< matrix.build_type >>
      - run-pytest:
          name: benchmarks
          build_type: release
          test_selection: performance
          run_in_parallel: false
          requires:
            - build-zenith-release
      - docker-image:
          # Context gives an ability to login
          context: Docker Hub
          # Build image only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - pg_regress-tests-release
            - other-tests-release
      - remote-ci-trigger:
          # Context passes credentials for gh api
          context: CI_ACCESS_TOKEN
          remote_repo: "zenithdb/console"
          requires:
            # XXX: Successful build doesn't mean everything is OK, but
            # the job to be triggered takes so much time to complete (~22 min)
            # that it's better not to wait for the commented-out steps
            - build-zenith-debug
            # - pg_regress-tests-release
            # - other-tests-release