diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml new file mode 100644 index 0000000000..94fac2ee99 --- /dev/null +++ b/.github/actions/run-python-test-set/action.yml @@ -0,0 +1,119 @@ +name: 'Run python test' +description: 'Runs a Neon python test set, performing all the required preparations before' + +inputs: + # Select the type of Rust build. Must be "release" or "debug". + build_type: + required: true + rust_toolchain: + required: true + # This parameter is required, to prevent the mistake of running all tests in one job. + test_selection: + required: true + # Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr + extra_params: + required: false + default: '' + needs_postgres_source: + required: false + default: 'false' + run_in_parallel: + required: false + default: 'true' + save_perf_report: + required: false + default: 'false' + +runs: + using: "composite" + steps: + - name: Get Neon artifact for restoration + uses: actions/download-artifact@v3 + with: + name: neon-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.rust_toolchain }}-artifact + path: ./neon-artifact/ + + - name: Extract Neon artifact + shell: bash -ex {0} + run: | + mkdir -p /tmp/neon/ + tar -xf ./neon-artifact/neon.tgz -C /tmp/neon/ + rm -rf ./neon-artifact/ + + - name: Checkout + if: inputs.needs_postgres_source == 'true' + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 + + - name: Cache poetry deps + id: cache_poetry + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry/virtualenvs + key: v1-${{ runner.os }}-python-deps-${{ hashFiles('poetry.lock') }} + + - name: Install Python deps + shell: bash -ex {0} + run: ./scripts/pysync + + - name: Run pytest + env: + ZENITH_BIN: /tmp/neon/bin + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + TEST_OUTPUT: /tmp/test_output + # this variable will be embedded in perf test report + # and is needed to distinguish different environments + PLATFORM: github-actions-selfhosted + shell: bash -ex {0} + run: | + PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)" + rm -rf $PERF_REPORT_DIR + + TEST_SELECTION="test_runner/${{ inputs.test_selection }}" + EXTRA_PARAMS="${{ inputs.extra_params }}" + if [ -z "$TEST_SELECTION" ]; then + echo "test_selection must be set" + exit 1 + fi + if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then + EXTRA_PARAMS="-n4 $EXTRA_PARAMS" + fi + if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then + if [[ "$GITHUB_REF" == "main" ]]; then + mkdir -p "$PERF_REPORT_DIR" + EXTRA_PARAMS="--out-dir $PERF_REPORT_DIR $EXTRA_PARAMS" + fi + fi + + if [[ "${{ inputs.build_type }}" == "debug" ]]; then + cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/neon/coverage run) + elif [[ "${{ inputs.build_type }}" == "release" ]]; then + cov_prefix=() + fi + + # Run the tests. + # + # The junit.xml file allows CircleCI to display more fine-grained test information + # in its "Tests" tab in the results page. + # --verbose prints name of each test (helpful when there are + # multiple tests in one file) + # -rA prints summary in the end + # -n4 uses four processes to run tests via pytest-xdist + # -s is not used to prevent pytest from capturing output, because tests are running + # in parallel and logs are mixed between different tests + "${cov_prefix[@]}" ./scripts/pytest \ + --junitxml=$TEST_OUTPUT/junit.xml \ + --tb=short \ + --verbose \ + -m "not remote_cluster" \ + -rA $TEST_SELECTION $EXTRA_PARAMS + + if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then + if [[ "$GITHUB_REF" == "main" ]]; then + export REPORT_FROM="$PERF_REPORT_DIR" + export REPORT_TO=local + scripts/generate_and_push_perf_report.sh + fi + fi diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 0000000000..5f4dd754d2 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,276 @@ +name: build_and_test +on: [ push ] +defaults: + run: + shell: bash -ex {0} + +jobs: + build-postgres: + runs-on: [ self-hosted, Linux, k8s-runner ] + strategy: + matrix: + build_type: [ debug, release ] + rust_toolchain: [ 1.58 ] + + env: + BUILD_TYPE: ${{ matrix.build_type }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 + + - name: Set pg revision for caching + id: pg_ver + run: echo ::set-output name=pg_rev::$(git rev-parse HEAD:vendor/postgres) + + - name: Cache postgres build + id: cache_pg + uses: actions/cache@v3 + with: + path: tmp_install/ + key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_ver.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Build postgres + if: steps.cache_pg.outputs.cache-hit != 'true' + run: COPT='-Werror' mold -run make postgres -j$(nproc) + + # actions/cache@v3 does not allow concurrently using the same cache across job steps, so use a separate cache + - name: Prepare postgres artifact + run: tar -C tmp_install/ -czf ./pg.tgz . + - name: Upload postgres artifact + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + if-no-files-found: error + name: postgres-${{ runner.os }}-${{ matrix.build_type }}-artifact + path: ./pg.tgz + + + build-neon: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ build-postgres ] + strategy: + matrix: + build_type: [ debug, release ] + rust_toolchain: [ 1.58 ] + + env: + BUILD_TYPE: ${{ matrix.build_type }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 + + - name: Get postgres artifact for restoration + uses: actions/download-artifact@v3 + with: + name: postgres-${{ runner.os }}-${{ matrix.build_type }}-artifact + path: ./postgres-artifact/ + - name: Extract postgres artifact + run: | + mkdir ./tmp_install/ + tar -xf ./postgres-artifact/pg.tgz -C ./tmp_install/ + rm -rf ./postgres-artifact/ + + - name: Cache cargo deps + id: cache_cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry/ + ~/.cargo/git/ + target/ + key: v2-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }} + + - name: Run cargo build + run: | + if [[ $BUILD_TYPE == "debug" ]]; then + cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/neon/coverage run) + CARGO_FLAGS= + elif [[ $BUILD_TYPE == "release" ]]; then + cov_prefix=() + CARGO_FLAGS="--release --features profiling" + fi + + export CACHEPOT_BUCKET=zenith-rust-cachepot + export RUSTC_WRAPPER=cachepot + export AWS_ACCESS_KEY_ID="${{ secrets.AWS_ACCESS_KEY_ID }}" + export AWS_SECRET_ACCESS_KEY="${{ secrets.AWS_SECRET_ACCESS_KEY }}" + export HOME=/home/runner + "${cov_prefix[@]}" mold -run cargo build $CARGO_FLAGS --features failpoints --bins --tests + cachepot -s + + - name: Run cargo test + run: | + export HOME=/home/runner + if [[ $BUILD_TYPE == "debug" ]]; then + cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/neon/coverage run) + CARGO_FLAGS= + elif [[ $BUILD_TYPE == "release" ]]; then + cov_prefix=() + CARGO_FLAGS=--release + fi + + "${cov_prefix[@]}" cargo test $CARGO_FLAGS + + - name: Install rust binaries + run: | + export HOME=/home/runner + if [[ $BUILD_TYPE == "debug" ]]; then + cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/neon/coverage run) + elif [[ $BUILD_TYPE == "release" ]]; then + cov_prefix=() + fi + + binaries=$( + "${cov_prefix[@]}" cargo metadata --format-version=1 --no-deps | + jq -r '.packages[].targets[] | select(.kind | index("bin")) | .name' + ) + + test_exe_paths=$( + "${cov_prefix[@]}" cargo test --message-format=json --no-run | + jq -r '.executable | select(. != null)' + ) + + mkdir -p /tmp/neon/bin + mkdir -p /tmp/neon/test_bin + mkdir -p /tmp/neon/etc + + # Install target binaries + for bin in $binaries; do + SRC=target/$BUILD_TYPE/$bin + DST=/tmp/neon/bin/$bin + cp $SRC $DST + echo $DST >> /tmp/neon/etc/binaries.list + done + + # Install test executables (for code coverage) + if [[ $BUILD_TYPE == "debug" ]]; then + for bin in $test_exe_paths; do + SRC=$bin + DST=/tmp/neon/test_bin/$(basename $bin) + cp $SRC $DST + echo $DST >> /tmp/neon/etc/binaries.list + done + fi + + - name: Install postgres binaries + run: cp -a tmp_install /tmp/neon/pg_install + + - name: Merge coverage data + run: | + export HOME=/home/runner + # This will speed up workspace uploads + if [[ $BUILD_TYPE == "debug" ]]; then + scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/neon/coverage merge + fi + + - name: Prepare neon artifact + run: tar -C /tmp/neon/ -czf ./neon.tgz . + + - name: Upload neon binaries + uses: actions/upload-artifact@v3 + with: + retention-days: 7 + if-no-files-found: error + name: neon-${{ runner.os }}-${{ matrix.build_type }}-${{ matrix.rust_toolchain }}-artifact + path: ./neon.tgz + + check-codestyle-python: + runs-on: [ self-hosted, Linux, k8s-runner ] + strategy: + matrix: + rust_toolchain: [ 1.58 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 + + - name: Cache poetry deps + id: cache_poetry + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry/virtualenvs + key: v1-${{ runner.os }}-python-deps-${{ hashFiles('poetry.lock') }} + + - name: Install Python deps + run: ./scripts/pysync + + - name: Run yapf to ensure code format + run: poetry run yapf --recursive --diff . + + - name: Run mypy to check types + run: poetry run mypy . + + pg_regress-tests: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ build-neon ] + strategy: + matrix: + build_type: [ debug, release ] + rust_toolchain: [ 1.58 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 2 + + - name: Pytest regress tests + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ matrix.build_type }} + rust_toolchain: ${{ matrix.rust_toolchain }} + test_selection: batch_pg_regress + needs_postgres_source: true + + other-tests: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ build-neon ] + strategy: + matrix: + build_type: [ debug, release ] + rust_toolchain: [ 1.58 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 2 + + - name: Pytest other tests + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ matrix.build_type }} + rust_toolchain: ${{ matrix.rust_toolchain }} + test_selection: batch_others + + benchmarks: + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ build-neon ] + strategy: + matrix: + build_type: [ release ] + rust_toolchain: [ 1.58 ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 2 + + - name: Pytest benchmarks + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ matrix.build_type }} + rust_toolchain: ${{ matrix.rust_toolchain }} + test_selection: performance + run_in_parallel: false + # save_perf_report: true