neon/.circleci/config.yml

version: 2.1

orbs:
  python: circleci/python@1.4.0

executors:
  zenith-build-executor:
    resource_class: xlarge
    docker:
      - image: cimg/rust:1.51.0

jobs:

  # A job to build postgres
  build-postgres:
    executor: zenith-build-executor
    steps:
        # Checkout the git repo (circleci doesn't have a flag to enable submodules here)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres

      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v02-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}

        # FIXME We could cache our own docker container, instead of installing packages every time.
      - run:
          name: apt install dependencies
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              sudo apt update
              sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libxml2-dev libcurl4-openssl-dev
            fi

        # Build postgres if the restore_cache didn't find a build.
        # `make` can't figure out whether the cache is valid, since
        # it only compares file timestamps.
      - run:
          name: build postgres
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              # "depth 1" saves some time by not cloning the whole repo
              git submodule update --init --depth 1
              make postgres
            fi

      - save_cache:
          name: Save postgres cache
          key: v02-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}
          paths:
            - tmp_install

  # A job to build zenith rust code
  build-zenith:
    executor: zenith-build-executor
    parameters:
      build_type:
        type: enum
        enum: ["debug", "release"]
    steps:
      - run:
          name: apt install dependencies
          command: |
            sudo apt update
            sudo apt install libssl-dev clang

        # Checkout the git repo (without submodules)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres

      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v02-postgres-cache-{{ checksum "/tmp/cache-key-postgres" }}

      - restore_cache:
          name: Restore rust cache
          keys:
            # Require an exact match. While an out of date cache might speed up the build,
            # there's no way to clean out old packages, so the cache grows every time something
            # changes.
            - v02-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}

        # Build the rust code, including test binaries
      - run:
          name: Rust build << parameters.build_type >>
          command: |
            BUILD_TYPE="<< parameters.build_type >>"
            if [[ $BUILD_TYPE == "debug" ]]; then
              echo "Build in debug mode"
              cargo build --bins --tests
            elif [[ $BUILD_TYPE == "release" ]]; then
              echo "Build in release mode"
              cargo build --release --bins --tests
            fi

      - save_cache:
          name: Save rust cache
          key: v02-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}
          paths:
            - ~/.cargo/registry
            - ~/.cargo/git
            - target

        # Run rust unit tests
        # FIXME: remove -p zenith_utils once integration tests are moved to python
      - run: cargo test -p zenith_utils

        # Install the rust binaries, for use by test jobs
        # `--locked` is required; otherwise, `cargo install` will ignore Cargo.lock.
        # FIXME: this is a really silly way to install; maybe we should just output
        # a tarball as an artifact? Or a .deb package?
      - run:
          name: cargo install
          command: |
            cargo install --debug --locked --root /tmp/zenith --path pageserver
            cargo install --debug --locked --root /tmp/zenith --path walkeeper
            cargo install --debug --locked --root /tmp/zenith --path zenith

        # Install the postgres binaries, for use by test jobs
        # FIXME: this is a silly way to do "install"; maybe just output a standard
        # postgres package, whatever the favored form is (tarball? .deb package?)
        # Note that pg_regress needs some build artifacts that probably aren't
        # in the usual package...?
      - run:
          name: postgres install
          command: |
            cp -a tmp_install /tmp/zenith/pg_install

        # Save the rust output binaries for other jobs in this workflow.
      - persist_to_workspace:
          root: /tmp/zenith
          paths:
            - "*"

  run-pytest:
    #description: "Run pytest"
    executor: python/default
    parameters:
      # pytest args to specify the tests to run.
      #
      # This can be a test file name, e.g. 'test_pgbench.py, or a subdirectory,
      # or '-k foobar' to run tests containing string 'foobar'. See pytest man page
      # section SPECIFYING TESTS / SELECTING TESTS for details.
      #
      # Select the type of Rust build. Must be "release" or "debug".
      build_type:
        type: string
        default: "debug"
      # This parameter is required, to prevent the mistake of running all tests in one job.
      test_selection:
        type: string
        default: ""
      # Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr
      extra_params:
        type: string
        default: ""
      needs_postgres_source:
        type: boolean
        default: false
    steps:
      - attach_workspace:
          at: /tmp/zenith
      - checkout
      - when:
          condition: << parameters.needs_postgres_source >>
          steps:
            - run: git submodule update --init --depth 1
      - run: pip install pytest psycopg2
      - run:
          name: Run pytest
          working_directory: test_runner
          environment:
            - ZENITH_BIN: /tmp/zenith/bin
            - POSTGRES_DISTRIB_DIR: /tmp/zenith/pg_install
            - TEST_OUTPUT: /tmp/test_output
          command: |
            TEST_SELECTION="<< parameters.test_selection >>"
            EXTRA_PARAMS="<< parameters.extra_params >>"
            if [ -z "$TEST_SELECTION" ]; then
              echo "test_selection must be set"
              exit 1
            fi
            # Run the tests.
            #
            # The junit.xml file allows CircleCI to display more fine-grained test information
            # in its "Tests" tab in the results page.
            pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short $TEST_SELECTION $EXTRA_PARAMS
      - run:
          # CircleCI artifacts are preserved one file at a time, so skipping
          # this step isn't a good idea. If you want to extract the
          # pageserver state, perhaps a tarball would be a better idea.
          name: Delete pageserver data
          when: always
          command: |
            du -sh /tmp/test_output/*
            for DIR in /tmp/test_output/*; do
              mv $DIR/repo/pageserver.log $DIR/ || true # ignore errors
              for PGDIR in $DIR/repo/pgdatadirs/pg?; do
                echo "PGDIR: $PGDIR"
                NEW_LOG="${PGDIR##*/}_log"
                mv $PGDIR/log "$DIR/$NEW_LOG" || true # ignore errors
              done
              echo "rm $DIR/repo"
              rm -rf $DIR/repo
            done
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
      # The store_test_results step tells CircleCI where to find the junit.xml file.
      - store_test_results:
          path: /tmp/test_output

workflows:
  build_and_test:
    jobs:
      - build-postgres
      - build-zenith:
          name: build-zenith-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          requires:
            - build-postgres
      - run-pytest:
          name: pg_regress tests << matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_pg_regress
          needs_postgres_source: true
          requires:
            - build-zenith-<< matrix.build_type >>
      - run-pytest:
          name: other tests << matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_others
          requires:
            - build-zenith-<< matrix.build_type >>