neon/.circleci/config.yml

version: 2.1

orbs:
  python: circleci/python@1.4.0

executors:
  zenith-build-executor:
    resource_class: xlarge
    docker:
      - image: cimg/rust:1.52.1

jobs:
  check-codestyle:
    executor: zenith-build-executor
    steps:
      - checkout

      - run:
          name: rustfmt
          when: always
          command: |
            cargo fmt --all -- --check

  # A job to build postgres
  build-postgres:
    executor: zenith-build-executor
    steps:
        # Checkout the git repo (circleci doesn't have a flag to enable submodules here)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres

      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}

        # FIXME We could cache our own docker container, instead of installing packages every time.
      - run:
          name: apt install dependencies
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              sudo apt update
              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev
            fi

        # Build postgres if the restore_cache didn't find a build.
        # `make` can't figure out whether the cache is valid, since
        # it only compares file timestamps.
      - run:
          name: build postgres
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              # "depth 1" saves some time by not cloning the whole repo
              git submodule update --init --depth 1
              make postgres
            fi

      - save_cache:
          name: Save postgres cache
          key: v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
          paths:
            - tmp_install

  # A job to build zenith rust code
  build-zenith:
    executor: zenith-build-executor
    parameters:
      build_type:
        type: enum
        enum: ["debug", "release"]
    steps:
      - run:
          name: apt install dependencies
          command: |
            sudo apt update
            sudo apt install libssl-dev clang

        # Checkout the git repo (without submodules)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres

      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v03-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}

      - restore_cache:
          name: Restore rust cache
          keys:
            # Require an exact match. While an out of date cache might speed up the build,
            # there's no way to clean out old packages, so the cache grows every time something
            # changes.
            - v03-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}

        # Build the rust code, including test binaries
      - run:
          name: Rust build << parameters.build_type >>
          command: |
            export CARGO_INCREMENTAL=0
            BUILD_TYPE="<< parameters.build_type >>"
            if [[ $BUILD_TYPE == "debug" ]]; then
              echo "Build in debug mode"
              cargo build --bins --tests
            elif [[ $BUILD_TYPE == "release" ]]; then
              echo "Build in release mode"
              cargo build --release --bins --tests
            fi

      - save_cache:
          name: Save rust cache
          key: v03-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}
          paths:
            - ~/.cargo/registry
            - ~/.cargo/git
            - target

        # Run rust unit tests
      - run: cargo test

        # Install the rust binaries, for use by test jobs
        # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
        # FIXME: this is a really silly way to install; maybe we should just output
        # a tarball as an artifact? Or a .deb package?
      - run:
          name: cargo install
          command: |
            export CARGO_INCREMENTAL=0
            BUILD_TYPE="<< parameters.build_type >>"
            if [[ $BUILD_TYPE == "debug" ]]; then
              echo "Install debug mode"
              CARGO_FLAGS="--debug"
            elif [[ $BUILD_TYPE == "release" ]]; then
              echo "Install release mode"
              # The default is release mode; there is no --release flag.
              CARGO_FLAGS=""
            fi
            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path pageserver
            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path walkeeper
            cargo install $CARGO_FLAGS --locked --root /tmp/zenith --path zenith

        # Install the postgres binaries, for use by test jobs
        # FIXME: this is a silly way to do "install"; maybe just output a standard
        # postgres package, whatever the favored form is (tarball? .deb package?)
        # Note that pg_regress needs some build artifacts that probably aren't
        # in the usual package...?
      - run:
          name: postgres install
          command: |
            cp -a tmp_install /tmp/zenith/pg_install

        # Save the rust output binaries for other jobs in this workflow.
      - persist_to_workspace:
          root: /tmp/zenith
          paths:
            - "*"

  run-pytest:
    #description: "Run pytest"
    executor: python/default
    parameters:
      # pytest args to specify the tests to run.
      #
      # This can be a test file name, e.g. 'test_pgbench.py, or a subdirectory,
      # or '-k foobar' to run tests containing string 'foobar'. See pytest man page
      # section SPECIFYING TESTS / SELECTING TESTS for details.
      #
      # Select the type of Rust build. Must be "release" or "debug".
      build_type:
        type: string
        default: "debug"
      # This parameter is required, to prevent the mistake of running all tests in one job.
      test_selection:
        type: string
        default: ""
      # Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr
      extra_params:
        type: string
        default: ""
      needs_postgres_source:
        type: boolean
        default: false
    steps:
      - attach_workspace:
          at: /tmp/zenith
      - checkout
      - when:
          condition: << parameters.needs_postgres_source >>
          steps:
            - run: git submodule update --init --depth 1
      - run:
          name: Install pipenv & deps
          working_directory: test_runner
          command: |
            pip install pipenv
            pipenv install
      - run:
          name: Run pytest
          working_directory: test_runner
          environment:
            - ZENITH_BIN: /tmp/zenith/bin
            - POSTGRES_DISTRIB_DIR: /tmp/zenith/pg_install
            - TEST_OUTPUT: /tmp/test_output
          command: |
            TEST_SELECTION="<< parameters.test_selection >>"
            EXTRA_PARAMS="<< parameters.extra_params >>"
            if [ -z "$TEST_SELECTION" ]; then
              echo "test_selection must be set"
              exit 1
            fi
            # Run the tests.
            #
            # The junit.xml file allows CircleCI to display more fine-grained test information
            # in its "Tests" tab in the results page.
            # -s prevents pytest from capturing output, which helps to see
            # what's going on if the test hangs
            # --verbose prints name of each test (helpful when there are
            # multiple tests in one file)
            # -rA prints summary in the end
            pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short -s --verbose -rA $TEST_SELECTION $EXTRA_PARAMS
      - run:
          # CircleCI artifacts are preserved one file at a time, so skipping
          # this step isn't a good idea. If you want to extract the
          # pageserver state, perhaps a tarball would be a better idea.
          name: Delete all data but logs
          when: always
          command: |
            du -sh /tmp/test_output/*
            find /tmp/test_output -type f ! -name "pg.log" ! -name "pageserver.log" ! -name "wal_acceptor.log" ! -name "regression.diffs" -delete
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
      # The store_test_results step tells CircleCI where to find the junit.xml file.
      - store_test_results:
          path: /tmp/test_output

  # Build zenithdb/zenith:latest image and push it to Docker hub
  docker-image:
    docker:
      - image: cimg/base:2021.04
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run:
          name: Init postgres submodule
          command: git submodule update --init --depth 1
      - run:
          name: Build and push Docker image
          command: |
            echo $DOCKER_PWD | docker login -u $DOCKER_LOGIN --password-stdin
            docker build -t zenithdb/zenith:latest . && docker push zenithdb/zenith:latest

workflows:
  build_and_test:
    jobs:
      - check-codestyle
      - build-postgres
      - build-zenith:
          name: build-zenith-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          requires:
            - build-postgres
      - run-pytest:
          name: pg_regress tests << matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_pg_regress
          needs_postgres_source: true
          requires:
            - build-zenith-<< matrix.build_type >>
      - run-pytest:
          name: other tests << matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_others
          requires:
            - build-zenith-<< matrix.build_type >>
      - run-pytest:
          name: benchmarks
          build_type: release
          test_selection: performance
          requires:
            - build-zenith-release
      - docker-image:
          # Context gives an ability to login
          context: Docker Hub
          # Build image only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - pg_regress tests release
            - other tests release