neon/.circleci/config.yml

version: 2.1

executors:
  neon-xlarge-executor:
    resource_class: xlarge
    docker:
      # NB: when changed, do not forget to update rust image tag in all Dockerfiles
      - image: zimg/rust:1.58
  neon-executor:
    docker:
      - image: zimg/rust:1.58

jobs:
  # A job to build postgres
  build-postgres:
    executor: neon-xlarge-executor
    parameters:
      build_type:
        type: enum
        enum: ["debug", "release"]
    environment:
      BUILD_TYPE: << parameters.build_type >>
    steps:
        # Checkout the git repo (circleci doesn't have a flag to enable submodules here)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Append makefile as it could change the way postgres is built.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
              git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres
              cat Makefile >> /tmp/cache-key-postgres

      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}

        # Build postgres if the restore_cache didn't find a build.
        # `make` can't figure out whether the cache is valid, since
        # it only compares file timestamps.
      - run:
          name: build postgres
          command: |
            if [ ! -e tmp_install/bin/postgres ]; then
              # "depth 1" saves some time by not cloning the whole repo
              git submodule update --init --depth 1
              # bail out on any warnings
              COPT='-Werror' mold -run make postgres -j$(nproc)
            fi

      - save_cache:
          name: Save postgres cache
          key: v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}
          paths:
            - tmp_install

  # A job to build Neon rust code
  build-neon:
    executor: neon-xlarge-executor
    parameters:
      build_type:
        type: enum
        enum: ["debug", "release"]
    environment:
      BUILD_TYPE: << parameters.build_type >>
    steps:
        # Checkout the git repo (without submodules)
      - checkout

        # Grab the postgres git revision to build a cache key.
        # Append makefile as it could change the way postgres is built.
        # Note this works even though the submodule hasn't been checkout out yet.
      - run:
          name: Get postgres cache key
          command: |
            git rev-parse HEAD:vendor/postgres > /tmp/cache-key-postgres
            cat Makefile >> /tmp/cache-key-postgres


      - restore_cache:
          name: Restore postgres cache
          keys:
            # Restore ONLY if the rev key matches exactly
            - v04-postgres-cache-<< parameters.build_type >>-{{ checksum "/tmp/cache-key-postgres" }}

      - restore_cache:
          name: Restore rust cache
          keys:
            # Require an exact match. While an out of date cache might speed up the build,
            # there's no way to clean out old packages, so the cache grows every time something
            # changes.
            - v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}

        # Build the rust code, including test binaries
      - run:
          name: Rust build << parameters.build_type >>
          command: |
            if [[ $BUILD_TYPE == "debug" ]]; then
              CARGO_FLAGS=
            elif [[ $BUILD_TYPE == "release" ]]; then
              CARGO_FLAGS="--release --features profiling"
            fi

            export CARGO_INCREMENTAL=0
            export CACHEPOT_BUCKET=zenith-rust-cachepot
            export RUSTC_WRAPPER=cachepot
            export AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}"
            export AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}"
            mold -run cargo build $CARGO_FLAGS --features failpoints --bins --tests
            cachepot -s

      - save_cache:
          name: Save rust cache
          key: v04-rust-cache-deps-<< parameters.build_type >>-{{ checksum "Cargo.lock" }}
          paths:
            - ~/.cargo/registry
            - ~/.cargo/git
            - target

        # Run rust unit tests
      - run:
          name: cargo test
          command: |
            if [[ $BUILD_TYPE == "debug" ]]; then
              CARGO_FLAGS=
            elif [[ $BUILD_TYPE == "release" ]]; then
              CARGO_FLAGS=--release
            fi

            cargo test $CARGO_FLAGS

        # Install the rust binaries, for use by test jobs
      - run:
          name: Install rust binaries
          command: |
            binaries=$(
              cargo metadata --format-version=1 --no-deps |
              jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name'
            )

            test_exe_paths=$(
              cargo test --message-format=json --no-run |
              jq -r '.executable | select(. != null)'
            )

            mkdir -p /tmp/zenith/bin
            mkdir -p /tmp/zenith/test_bin
            mkdir -p /tmp/zenith/etc

            # Install target binaries
            for bin in $binaries; do
              SRC=target/$BUILD_TYPE/$bin
              DST=/tmp/zenith/bin/$bin
              cp $SRC $DST
            done

        # Install the postgres binaries, for use by test jobs
      - run:
          name: Install postgres binaries
          command: |
            cp -a tmp_install /tmp/zenith/pg_install

      # Save rust binaries for other jobs in the workflow
      - persist_to_workspace:
          root: /tmp/zenith
          paths:
            - "*"

  check-codestyle-python:
    executor: neon-executor
    steps:
      - checkout
      - restore_cache:
          keys:
            - v2-python-deps-{{ checksum "poetry.lock" }}
      - run:
          name: Install deps
          command: ./scripts/pysync
      - save_cache:
          key: v2-python-deps-{{ checksum "poetry.lock" }}
          paths:
            - /home/circleci/.cache/pypoetry/virtualenvs
      - run:
          name: Print versions
          when: always
          command: |
              poetry run python --version
              poetry show
      - run:
          name: Run yapf to ensure code format
          when: always
          command: poetry run yapf --recursive --diff .
      - run:
          name: Run mypy to check types
          when: always
          command: poetry run mypy .

  run-pytest:
    executor: neon-executor
    parameters:
      # pytest args to specify the tests to run.
      #
      # This can be a test file name, e.g. 'test_pgbench.py, or a subdirectory,
      # or '-k foobar' to run tests containing string 'foobar'. See pytest man page
      # section SPECIFYING TESTS / SELECTING TESTS for details.
      #
      # Select the type of Rust build. Must be "release" or "debug".
      build_type:
        type: string
        default: "debug"
      # This parameter is required, to prevent the mistake of running all tests in one job.
      test_selection:
        type: string
        default: ""
      # Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr
      extra_params:
        type: string
        default: ""
      needs_postgres_source:
        type: boolean
        default: false
      run_in_parallel:
        type: boolean
        default: true
      save_perf_report:
        type: boolean
        default: false
    environment:
      BUILD_TYPE: << parameters.build_type >>
    steps:
      - attach_workspace:
          at: /tmp/zenith
      - checkout
      - when:
          condition: << parameters.needs_postgres_source >>
          steps:
            - run: git submodule update --init --depth 1
      - restore_cache:
          keys:
            - v2-python-deps-{{ checksum "poetry.lock" }}
      - run:
          name: Install deps
          command: ./scripts/pysync
      - save_cache:
          key: v2-python-deps-{{ checksum "poetry.lock" }}
          paths:
            - /home/circleci/.cache/pypoetry/virtualenvs
      - run:
          name: Run pytest
          # pytest doesn't output test logs in real time, so CI job may fail with
          # `Too long with no output` error, if a test is running for a long time.
          # In that case, tests should have internal timeouts that are less than
          # no_output_timeout, specified here.
          no_output_timeout: 10m
          environment:
            - NEON_BIN: /tmp/zenith/bin
            - POSTGRES_DISTRIB_DIR: /tmp/zenith/pg_install
            - TEST_OUTPUT: /tmp/test_output
            # this variable will be embedded in perf test report
            # and is needed to distinguish different environments
            - PLATFORM: zenith-local-ci
          command: |
            PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)"
            rm -rf $PERF_REPORT_DIR

            TEST_SELECTION="test_runner/<< parameters.test_selection >>"
            EXTRA_PARAMS="<< parameters.extra_params >>"
            if [ -z "$TEST_SELECTION" ]; then
              echo "test_selection must be set"
              exit 1
            fi
            if << parameters.run_in_parallel >>; then
              EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
            fi
            if << parameters.save_perf_report >>; then
              if [[ $CIRCLE_BRANCH == "main" ]]; then
                mkdir -p "$PERF_REPORT_DIR"
                EXTRA_PARAMS="--out-dir $PERF_REPORT_DIR $EXTRA_PARAMS"
              fi
            fi

            export GITHUB_SHA=$CIRCLE_SHA1

            # Run the tests.
            #
            # The junit.xml file allows CircleCI to display more fine-grained test information
            # in its "Tests" tab in the results page.
            # --verbose prints name of each test (helpful when there are
            # multiple tests in one file)
            # -rA prints summary in the end
            # -n4 uses four processes to run tests via pytest-xdist
            # -s is not used to prevent pytest from capturing output, because tests are running
            # in parallel and logs are mixed between different tests
            ./scripts/pytest \
              --junitxml=$TEST_OUTPUT/junit.xml \
              --tb=short \
              --verbose \
              -m "not remote_cluster" \
              -rA $TEST_SELECTION $EXTRA_PARAMS

            if << parameters.save_perf_report >>; then
              if [[ $CIRCLE_BRANCH == "main" ]]; then
                export REPORT_FROM="$PERF_REPORT_DIR"
                export REPORT_TO=local
                scripts/generate_and_push_perf_report.sh
              fi
            fi
      - run:
          # CircleCI artifacts are preserved one file at a time, so skipping
          # this step isn't a good idea. If you want to extract the
          # pageserver state, perhaps a tarball would be a better idea.
          name: Delete all data but logs
          when: always
          command: |
            du -sh /tmp/test_output/*
            find /tmp/test_output -type f ! -name "*.log" ! -name "regression.diffs" ! -name "junit.xml" ! -name "*.filediff" ! -name "*.stdout" ! -name "*.stderr" ! -name "flamegraph.svg" ! -name "*.metrics" -delete
            du -sh /tmp/test_output/*
      - store_artifacts:
          path: /tmp/test_output
      # The store_test_results step tells CircleCI where to find the junit.xml file.
      - store_test_results:
          path: /tmp/test_output
      # Save data (if any)
      - persist_to_workspace:
          root: /tmp/zenith
          paths:
            - "*"

  # Build neondatabase/neon:latest image and push it to Docker hub
  docker-image:
    docker:
      - image: cimg/base:2021.04
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run:
          name: Init postgres submodule
          command: git submodule update --init --depth 1
      - run:
          name: Build and push Docker image
          command: |
            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
            DOCKER_TAG=$(git log --oneline|wc -l)
            docker build \
              --pull \
              --build-arg GIT_VERSION=${CIRCLE_SHA1} \
              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
              --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:latest .
            docker push neondatabase/neon:${DOCKER_TAG}
            docker push neondatabase/neon:latest

  # Build neondatabase/compute-node:latest image and push it to Docker hub
  docker-image-compute:
    docker:
      - image: cimg/base:2021.04
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run:
          name: Build and push compute-tools Docker image
          command: |
            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
            docker build \
              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
              --tag neondatabase/compute-tools:local \
              --tag neondatabase/compute-tools:latest \
              -f Dockerfile.compute-tools .
            # Only push :latest image
            docker push neondatabase/compute-tools:latest
      - run:
          name: Init postgres submodule
          command: git submodule update --init --depth 1
      - run:
          name: Build and push compute-node Docker image
          command: |
            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
            DOCKER_TAG=$(git log --oneline|wc -l)
            docker build --tag neondatabase/compute-node:${DOCKER_TAG} \
              --tag neondatabase/compute-node:latest vendor/postgres \
              --build-arg COMPUTE_TOOLS_TAG=local
            docker push neondatabase/compute-node:${DOCKER_TAG}
            docker push neondatabase/compute-node:latest

  # Build production neondatabase/neon:release image and push it to Docker hub
  docker-image-release:
    docker:
      - image: cimg/base:2021.04
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run:
          name: Init postgres submodule
          command: git submodule update --init --depth 1
      - run:
          name: Build and push Docker image
          command: |
            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
            DOCKER_TAG="release-$(git log --oneline|wc -l)"
            docker build \
              --pull \
              --build-arg GIT_VERSION=${CIRCLE_SHA1} \
              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
              --tag neondatabase/neon:${DOCKER_TAG} --tag neondatabase/neon:release .
            docker push neondatabase/neon:${DOCKER_TAG}
            docker push neondatabase/neon:release

  # Build production neondatabase/compute-node:release image and push it to Docker hub
  docker-image-compute-release:
    docker:
      - image: cimg/base:2021.04
    steps:
      - checkout
      - setup_remote_docker:
          docker_layer_caching: true
      - run:
          name: Build and push compute-tools Docker image
          command: |
            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
            docker build \
              --build-arg AWS_ACCESS_KEY_ID="${CACHEPOT_AWS_ACCESS_KEY_ID}" \
              --build-arg AWS_SECRET_ACCESS_KEY="${CACHEPOT_AWS_SECRET_ACCESS_KEY}" \
              --tag neondatabase/compute-tools:release \
              --tag neondatabase/compute-tools:local \
              -f Dockerfile.compute-tools .
            # Only push :release image
            docker push neondatabase/compute-tools:release
      - run:
          name: Init postgres submodule
          command: git submodule update --init --depth 1
      - run:
          name: Build and push compute-node Docker image
          command: |
            echo $NEON_DOCKER_PWD | docker login -u $NEON_DOCKER_LOGIN --password-stdin
            DOCKER_TAG="release-$(git log --oneline|wc -l)"
            docker build --tag neondatabase/compute-node:${DOCKER_TAG} \
              --tag neondatabase/compute-node:release vendor/postgres \
              --build-arg COMPUTE_TOOLS_TAG=local
            docker push neondatabase/compute-node:${DOCKER_TAG}
            docker push neondatabase/compute-node:release

  deploy-staging:
    docker:
      - image: cimg/python:3.10
    steps:
      - checkout
      - setup_remote_docker
      - run:
          name: Setup ansible
          command: |
            pip install --progress-bar off --user ansible boto3
      - run:
          name: Redeploy
          command: |
            cd "$(pwd)/.circleci/ansible"

            ./get_binaries.sh

            echo "${TELEPORT_SSH_KEY}"  | tr -d '\n'| base64 --decode >ssh-key
            echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
            chmod 0600 ssh-key
            ssh-add ssh-key
            rm -f ssh-key ssh-key-cert.pub

            ansible-playbook deploy.yaml -i staging.hosts
            rm -f neon_install.tar.gz .neon_current_version

  deploy-staging-proxy:
    docker:
      - image: cimg/base:2021.04
    environment:
      KUBECONFIG: .kubeconfig
    steps:
      - checkout
      - run:
          name: Store kubeconfig file
          command: |
            echo "${STAGING_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG}
            chmod 0600 ${KUBECONFIG}
      - run:
          name: Setup helm v3
          command: |
            curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
            helm repo add neondatabase https://neondatabase.github.io/helm-charts
      - run:
          name: Re-deploy proxy
          command: |
            DOCKER_TAG=$(git log --oneline|wc -l)
            helm upgrade neon-proxy       neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
            helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/staging.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait

  deploy-neon-stress:
    docker:
      - image: cimg/python:3.10
    steps:
      - checkout
      - setup_remote_docker
      - run:
          name: Setup ansible
          command: |
            pip install --progress-bar off --user ansible boto3
      - run:
          name: Redeploy
          command: |
            cd "$(pwd)/.circleci/ansible"

            ./get_binaries.sh

            echo "${TELEPORT_SSH_KEY}"  | tr -d '\n'| base64 --decode >ssh-key
            echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
            chmod 0600 ssh-key
            ssh-add ssh-key
            rm -f ssh-key ssh-key-cert.pub

            ansible-playbook deploy.yaml -i neon-stress.hosts
            rm -f neon_install.tar.gz .neon_current_version

  deploy-neon-stress-proxy:
    docker:
      - image: cimg/base:2021.04
    environment:
      KUBECONFIG: .kubeconfig
    steps:
      - checkout
      - run:
          name: Store kubeconfig file
          command: |
            echo "${NEON_STRESS_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG}
            chmod 0600 ${KUBECONFIG}
      - run:
          name: Setup helm v3
          command: |
            curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
            helm repo add neondatabase https://neondatabase.github.io/helm-charts
      - run:
          name: Re-deploy proxy
          command: |
            DOCKER_TAG=$(git log --oneline|wc -l)
            helm upgrade neon-stress-proxy       neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
            helm upgrade neon-stress-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/neon-stress.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait

  deploy-release:
    docker:
      - image: cimg/python:3.10
    steps:
      - checkout
      - setup_remote_docker
      - run:
          name: Setup ansible
          command: |
            pip install --progress-bar off --user ansible boto3
      - run:
          name: Redeploy
          command: |
            cd "$(pwd)/.circleci/ansible"

            RELEASE=true ./get_binaries.sh

            echo "${TELEPORT_SSH_KEY}"  | tr -d '\n'| base64 --decode >ssh-key
            echo "${TELEPORT_SSH_CERT}" | tr -d '\n'| base64 --decode >ssh-key-cert.pub
            chmod 0600 ssh-key
            ssh-add ssh-key
            rm -f ssh-key ssh-key-cert.pub

            ansible-playbook deploy.yaml -i production.hosts
            rm -f neon_install.tar.gz .neon_current_version

  deploy-release-proxy:
    docker:
      - image: cimg/base:2021.04
    environment:
      KUBECONFIG: .kubeconfig
    steps:
      - checkout
      - run:
          name: Store kubeconfig file
          command: |
            echo "${PRODUCTION_KUBECONFIG_DATA}" | base64 --decode > ${KUBECONFIG}
            chmod 0600 ${KUBECONFIG}
      - run:
          name: Setup helm v3
          command: |
            curl -s https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
            helm repo add neondatabase https://neondatabase.github.io/helm-charts
      - run:
          name: Re-deploy proxy
          command: |
            DOCKER_TAG="release-$(git log --oneline|wc -l)"
            helm upgrade neon-proxy       neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy.yaml --set image.tag=${DOCKER_TAG} --wait
            helm upgrade neon-proxy-scram neondatabase/neon-proxy --install -f .circleci/helm-values/production.proxy-scram.yaml --set image.tag=${DOCKER_TAG} --wait

workflows:
  build_and_test:
    jobs:
      - check-codestyle-python
      - build-postgres:
          name: build-postgres-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
      - build-neon:
          name: build-neon-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          requires:
            - build-postgres-<< matrix.build_type >>
      - run-pytest:
          name: pg_regress-tests-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_pg_regress
          needs_postgres_source: true
          requires:
            - build-neon-<< matrix.build_type >>
      - run-pytest:
          name: other-tests-<< matrix.build_type >>
          matrix:
            parameters:
              build_type: ["debug", "release"]
          test_selection: batch_others
          requires:
            - build-neon-<< matrix.build_type >>
      - run-pytest:
          name: benchmarks
          context: PERF_TEST_RESULT_CONNSTR
          build_type: release
          test_selection: performance
          run_in_parallel: false
          save_perf_report: true
          requires:
            - build-neon-release
      - docker-image:
          # Context gives an ability to login
          context: Docker Hub
          # Build image only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - pg_regress-tests-release
            - other-tests-release
      - docker-image-compute:
          # Context gives an ability to login
          context: Docker Hub
          # Build image only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - pg_regress-tests-release
            - other-tests-release
      - deploy-staging:
          # Context gives an ability to login
          context: Docker Hub
          # deploy only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - docker-image
      - deploy-staging-proxy:
          # deploy only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - docker-image

      - deploy-neon-stress:
          # Context gives an ability to login
          context: Docker Hub
          # deploy only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - docker-image
      - deploy-neon-stress-proxy:
          # deploy only for commits to main
          filters:
            branches:
              only:
                - main
          requires:
            - docker-image

      - docker-image-release:
          # Context gives an ability to login
          context: Docker Hub
          # Build image only for commits to main
          filters:
            branches:
              only:
                - release
          requires:
            - pg_regress-tests-release
            - other-tests-release
      - docker-image-compute-release:
          # Context gives an ability to login
          context: Docker Hub
          # Build image only for commits to main
          filters:
            branches:
              only:
                - release
          requires:
            - pg_regress-tests-release
            - other-tests-release
      - deploy-release:
          # Context gives an ability to login
          context: Docker Hub
          # deploy only for commits to main
          filters:
            branches:
              only:
                - release
          requires:
            - docker-image-release
      - deploy-release-proxy:
          # deploy only for commits to main
          filters:
            branches:
              only:
                - release
          requires:
            - docker-image-release