From 0aa2f5c9a5eab3d507c3c468d0902fe4e91de863 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Mon, 12 Dec 2022 12:58:55 +0200 Subject: [PATCH] Regroup CI testing (#3049) Part of https://github.com/neondatabase/neon/pull/2410 and https://github.com/neondatabase/neon/pull/2407 * adds `hashFiles('rust-toolchain.toml')` into Rust cache keys, thus removing one of the manual steps to do when upgrading rustc * copies Python and Rust style checks from the `codestyle.yml` workflow * adjusts shell defaults in the main workflow * replaces `codestyle.yml` with a `neon_extra_builds.yml` worlflow The new workflow runs on commits to `main` (`codestyle.yml` was run per PR), and runs two custom builds on GH agents: * macos-latest, to ensure the entire project compiles on it (no tests run) There were no frequent breakages on macOs in our builds, so we can check it rarely without making every storage PR to wait for it to complete. The updated mac build use release builds now, so presumably should work a bit faster due to overall smaller files to cache between builds. * ubuntu-latest, without caches, to produce full compilation stats for Rust builds and upload it as an artifact to GitHub Old `clippy build --timings` stats were collected from the builds that use caches and incremental calculation hence never could produce a full report, it got removed. --- .github/workflows/build_and_test.yml | 132 ++++++++++++++----- .github/workflows/codestyle.yml | 166 ------------------------ .github/workflows/neon_extra_builds.yml | 128 ++++++++++++++++++ rust-toolchain.toml | 2 +- 4 files changed, 228 insertions(+), 200 deletions(-) delete mode 100644 .github/workflows/codestyle.yml create mode 100644 .github/workflows/neon_extra_builds.yml diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 080512fa14..44b691754a 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -7,6 +7,10 @@ on: - release pull_request: +defaults: + run: + shell: bash -euxo pipefail {0} + concurrency: # Allow only one workflow per any non-`main` branch. group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }} @@ -45,6 +49,79 @@ jobs: shell: bash id: build-tag + check-codestyle-python: + runs-on: [ self-hosted, Linux, k8s-runner ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: false + fetch-depth: 1 + + - name: Cache poetry deps + id: cache_poetry + uses: actions/cache@v3 + with: + path: ~/.cache/pypoetry/virtualenvs + key: v1-codestyle-python-deps-${{ hashFiles('poetry.lock') }} + + - name: Install Python deps + run: ./scripts/pysync + + - name: Run isort to ensure code format + run: poetry run isort --diff --check . + + - name: Run black to ensure code format + run: poetry run black --diff --check . + + - name: Run flake8 to ensure code format + run: poetry run flake8 . + + - name: Run mypy to check types + run: poetry run mypy . + + check-codestyle-rust: + runs-on: [ self-hosted, dev, x64 ] + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 + + - name: Restore cargo deps cache + id: cache_cargo + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry/ + !~/.cargo/registry/src + ~/.cargo/git/ + target/ + key: v1-${{ runner.os }}-cargo-clippy-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} + + # Some of our rust modules use FFI and need those to be checked + - name: Get postgres headers + run: make postgres-headers -j$(nproc) + - name: Run cargo clippy + run: ./run_clippy.sh + + # Use `${{ !cancelled() }}` to run quck tests after the longer clippy run + - name: Check formatting + if: ${{ !cancelled() }} + run: cargo fmt --all -- --check + + # https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci + - name: Check rust dependencies + if: ${{ !cancelled() }} + run: | + cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date + cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack + build-neon: runs-on: [ self-hosted, dev, x64 ] container: @@ -79,12 +156,10 @@ jobs: - name: Set pg 14 revision for caching id: pg_v14_rev run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT - shell: bash -euxo pipefail {0} - name: Set pg 15 revision for caching id: pg_v15_rev run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT - shell: bash -euxo pipefail {0} # Set some environment variables used by all the steps. # @@ -101,16 +176,15 @@ jobs: if [[ $BUILD_TYPE == "debug" ]]; then cov_prefix="scripts/coverage --profraw-prefix=$GITHUB_JOB --dir=/tmp/coverage run" CARGO_FEATURES="--features testing" - CARGO_FLAGS="--locked --timings $CARGO_FEATURES" + CARGO_FLAGS="--locked $CARGO_FEATURES" elif [[ $BUILD_TYPE == "release" ]]; then cov_prefix="" CARGO_FEATURES="--features testing,profiling" - CARGO_FLAGS="--locked --timings --release $CARGO_FEATURES" + CARGO_FLAGS="--locked --release $CARGO_FEATURES" fi echo "cov_prefix=${cov_prefix}" >> $GITHUB_ENV echo "CARGO_FEATURES=${CARGO_FEATURES}" >> $GITHUB_ENV echo "CARGO_FLAGS=${CARGO_FLAGS}" >> $GITHUB_ENV - shell: bash -euxo pipefail {0} # Don't include the ~/.cargo/registry/src directory. It contains just # uncompressed versions of the crates in ~/.cargo/registry/cache @@ -127,8 +201,8 @@ jobs: target/ # Fall back to older versions of the key, if no cache for current Cargo.lock was found key: | - v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }} - v10-${{ runner.os }}-${{ matrix.build_type }}-cargo- + v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} + v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}- - name: Cache postgres v14 build id: cache_pg_14 @@ -147,26 +221,21 @@ jobs: - name: Build postgres v14 if: steps.cache_pg_14.outputs.cache-hit != 'true' run: mold -run make postgres-v14 -j$(nproc) - shell: bash -euxo pipefail {0} - name: Build postgres v15 if: steps.cache_pg_15.outputs.cache-hit != 'true' run: mold -run make postgres-v15 -j$(nproc) - shell: bash -euxo pipefail {0} - name: Build neon extensions run: mold -run make neon-pg-ext -j$(nproc) - shell: bash -euxo pipefail {0} - name: Run cargo build run: | ${cov_prefix} mold -run cargo build $CARGO_FLAGS --bins --tests - shell: bash -euxo pipefail {0} - name: Run cargo test run: | ${cov_prefix} cargo test $CARGO_FLAGS - shell: bash -euxo pipefail {0} - name: Install rust binaries run: | @@ -207,11 +276,9 @@ jobs: echo "/tmp/neon/bin/$bin" >> /tmp/coverage/binaries.list done fi - shell: bash -euxo pipefail {0} - name: Install postgres binaries run: cp -a pg_install /tmp/neon/pg_install - shell: bash -euxo pipefail {0} - name: Upload Neon artifact uses: ./.github/actions/upload @@ -219,17 +286,6 @@ jobs: name: neon-${{ runner.os }}-${{ matrix.build_type }}-artifact path: /tmp/neon - - name: Prepare cargo build timing stats for storing - run: | - mkdir -p "/tmp/neon/cargo-timings/$BUILD_TYPE/" - cp -r ./target/cargo-timings/* "/tmp/neon/cargo-timings/$BUILD_TYPE/" - shell: bash -euxo pipefail {0} - - name: Upload cargo build stats - uses: ./.github/actions/upload - with: - name: neon-${{ runner.os }}-${{ matrix.build_type }}-build-stats - path: /tmp/neon/cargo-timings/ - # XXX: keep this after the binaries.list is formed, so the coverage can properly work later - name: Merge and upload coverage data if: matrix.build_type == 'debug' @@ -250,7 +306,7 @@ jobs: uses: actions/checkout@v3 with: submodules: true - fetch-depth: 2 + fetch-depth: 1 - name: Pytest regression tests uses: ./.github/actions/run-python-test-set @@ -284,7 +340,7 @@ jobs: uses: actions/checkout@v3 with: submodules: true - fetch-depth: 2 + fetch-depth: 1 - name: Pytest benchmarks uses: ./.github/actions/run-python-test-set @@ -330,7 +386,6 @@ jobs: SHA: ${{ github.event.pull_request.head.sha || github.sha }} REPORT_URL: ${{ steps.create-allure-report.outputs.report-url }} TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR }} - shell: bash -euxo pipefail {0} run: | curl --fail --output suites.json ${REPORT_URL%/index.html}/data/suites.json ./scripts/pysync @@ -363,7 +418,7 @@ jobs: !~/.cargo/registry/src ~/.cargo/git/ target/ - key: v10-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('Cargo.lock') }} + key: v1-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ hashFiles('rust-toolchain.toml') }}-${{ hashFiles('Cargo.lock') }} - name: Get Neon artifact uses: ./.github/actions/download @@ -379,7 +434,6 @@ jobs: - name: Merge coverage data run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge - shell: bash -euxo pipefail {0} - name: Build and upload coverage report run: | @@ -412,7 +466,6 @@ jobs: \"description\": \"Coverage report is ready\", \"target_url\": \"$REPORT_URL\" }" - shell: bash -euxo pipefail {0} trigger-e2e-tests: runs-on: [ self-hosted, dev, x64 ] @@ -463,6 +516,9 @@ jobs: runs-on: [ self-hosted, dev, x64 ] needs: [ tag ] container: gcr.io/kaniko-project/executor:v1.9.0-debug + defaults: + run: + shell: sh -eu {0} steps: - name: Checkout @@ -481,6 +537,9 @@ jobs: runs-on: [ self-hosted, dev, x64 ] needs: [ tag ] container: gcr.io/kaniko-project/executor:v1.9.0-debug + defaults: + run: + shell: sh -eu {0} steps: - name: Checkout @@ -496,6 +555,10 @@ jobs: runs-on: [ self-hosted, dev, x64 ] container: gcr.io/kaniko-project/executor:v1.9.0-debug needs: [ tag ] + defaults: + run: + shell: sh -eu {0} + steps: - name: Checkout uses: actions/checkout@v1 # v3 won't work with kaniko @@ -513,6 +576,10 @@ jobs: runs-on: [ self-hosted, dev, x64 ] container: gcr.io/kaniko-project/executor:v1.9.0-debug needs: [ tag ] + defaults: + run: + shell: sh -eu {0} + steps: - name: Checkout uses: actions/checkout@v1 # v3 won't work with kaniko @@ -780,7 +847,7 @@ jobs: # If it notices a fresh storage it may bump the compute version. And if compute image failed to build it may break things badly needs: [ push-docker-hub, tag, regress-tests ] if: | - contains(github.event.pull_request.labels.*.name, 'deploy-test-storage') && + contains(github.event.pull_request.labels.*.name, 'deploy-test-storage') && github.event_name != 'workflow_dispatch' defaults: run: @@ -1100,7 +1167,6 @@ jobs: if: github.ref_name == 'release' && github.event_name != 'workflow_dispatch' steps: - name: Promote compatibility snapshot for the release - shell: bash -euxo pipefail {0} env: BUCKET: neon-github-public-dev PREFIX: artifacts/latest diff --git a/.github/workflows/codestyle.yml b/.github/workflows/codestyle.yml deleted file mode 100644 index 01fef71c9a..0000000000 --- a/.github/workflows/codestyle.yml +++ /dev/null @@ -1,166 +0,0 @@ -name: Check code style and build - -on: - push: - branches: - - main - pull_request: - -defaults: - run: - shell: bash -euxo pipefail {0} - -concurrency: - # Allow only one workflow per any non-`main` branch. - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }} - cancel-in-progress: true - -env: - RUST_BACKTRACE: 1 - COPT: '-Werror' - -jobs: - check-codestyle-rust: - strategy: - fail-fast: false - matrix: - # XXX: both OSes have rustup - # * https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md#rust-tools - # * https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md#rust-tools - # this is all we need to install our toolchain later via rust-toolchain.toml - # so don't install any toolchain explicitly. - os: [ubuntu-latest, macos-latest] - timeout-minutes: 90 - name: check codestyle rust and postgres - runs-on: ${{ matrix.os }} - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 2 - - - name: Check formatting - run: cargo fmt --all -- --check - - - name: Install Ubuntu postgres dependencies - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler - - - name: Install macOS postgres dependencies - if: matrix.os == 'macos-latest' - run: brew install flex bison openssl protobuf - - - name: Set pg 14 revision for caching - id: pg_v14_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT - shell: bash -euxo pipefail {0} - - - name: Set pg 15 revision for caching - id: pg_v15_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT - shell: bash -euxo pipefail {0} - - - name: Cache postgres v14 build - id: cache_pg_14 - uses: actions/cache@v3 - with: - path: pg_install/v14 - key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - - - name: Cache postgres v15 build - id: cache_pg_15 - uses: actions/cache@v3 - with: - path: pg_install/v15 - key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - - - name: Set extra env for macOS - if: matrix.os == 'macos-latest' - run: | - echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV - echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV - - - name: Build postgres v14 - if: steps.cache_pg_14.outputs.cache-hit != 'true' - run: make postgres-v14 - shell: bash -euxo pipefail {0} - - - name: Build postgres v15 - if: steps.cache_pg_15.outputs.cache-hit != 'true' - run: make postgres-v15 - shell: bash -euxo pipefail {0} - - - name: Build neon extensions - run: make neon-pg-ext - - - name: Cache cargo deps - id: cache_cargo - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - !~/.cargo/registry/src - ~/.cargo/git - target - key: v6-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-rust - - - name: Run cargo clippy - run: ./run_clippy.sh - - - name: Ensure all project builds - run: cargo build --locked --all --all-targets - - check-rust-dependencies: - runs-on: [ self-hosted, dev, x64 ] - container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned - options: --init - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - # https://github.com/facebookincubator/cargo-guppy/tree/bec4e0eb29dcd1faac70b1b5360267fc02bf830e/tools/cargo-hakari#2-keep-the-workspace-hack-up-to-date-in-ci - - name: Check every project module is covered by Hakari - run: | - cargo hakari generate --diff # workspace-hack Cargo.toml is up-to-date - cargo hakari manage-deps --dry-run # all workspace crates depend on workspace-hack - shell: bash -euxo pipefail {0} - - check-codestyle-python: - runs-on: [ self-hosted, Linux, k8s-runner ] - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: false - fetch-depth: 1 - - - name: Cache poetry deps - id: cache_poetry - uses: actions/cache@v3 - with: - path: ~/.cache/pypoetry/virtualenvs - key: v1-codestyle-python-deps-${{ hashFiles('poetry.lock') }} - - - name: Install Python deps - run: ./scripts/pysync - - - name: Run isort to ensure code format - run: poetry run isort --diff --check . - - - name: Run black to ensure code format - run: poetry run black --diff --check . - - - name: Run flake8 to ensure code format - run: poetry run flake8 . - - - name: Run mypy to check types - run: poetry run mypy . diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml new file mode 100644 index 0000000000..b8600e0665 --- /dev/null +++ b/.github/workflows/neon_extra_builds.yml @@ -0,0 +1,128 @@ +name: Check neon with extra platform builds + +on: + push: + branches: + - main + +defaults: + run: + shell: bash -euxo pipefail {0} + +concurrency: + # Allow only one workflow per any non-`main` branch. + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref == 'refs/heads/main' && github.sha || 'anysha' }} + cancel-in-progress: true + +env: + RUST_BACKTRACE: 1 + COPT: '-Werror' + +jobs: + check-macos-build: + timeout-minutes: 90 + runs-on: macos-latest + + env: + # Use release build only, to have less debug info around + # Hence keeping target/ (and general cache size) smaller + BUILD_TYPE: release + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 + + - name: Install macOS postgres dependencies + run: brew install flex bison openssl protobuf + + - name: Set pg 14 revision for caching + id: pg_v14_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT + + - name: Set pg 15 revision for caching + id: pg_v15_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT + + - name: Cache postgres v14 build + id: cache_pg_14 + uses: actions/cache@v3 + with: + path: pg_install/v14 + key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Cache postgres v15 build + id: cache_pg_15 + uses: actions/cache@v3 + with: + path: pg_install/v15 + key: v1-${{ runner.os }}-${{ matrix.build_type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Set extra env for macOS + run: | + echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV + echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV + + - name: Cache cargo deps + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + !~/.cargo/registry/src + ~/.cargo/git + target + key: v1-${{ runner.os }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust + + - name: Build postgres v14 + if: steps.cache_pg_14.outputs.cache-hit != 'true' + run: make postgres-v14 -j$(nproc) + + - name: Build postgres v15 + if: steps.cache_pg_15.outputs.cache-hit != 'true' + run: make postgres-v15 -j$(nproc) + + - name: Build neon extensions + run: make neon-pg-ext -j$(nproc) + + - name: Run cargo build + run: cargo build --all --release + + - name: Check that no warnings are produced + run: ./run_clippy.sh + + gather-rust-build-stats: + timeout-minutes: 90 + runs-on: ubuntu-latest + + env: + BUILD_TYPE: release + # build with incremental compilation produce partial results + # so do not attempt to cache this build, also disable the incremental compilation + CARGO_INCREMENTAL: 0 + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: true + fetch-depth: 1 + + - name: Install Ubuntu postgres dependencies + run: | + sudo apt update + sudo apt install build-essential libreadline-dev zlib1g-dev flex bison libseccomp-dev libssl-dev protobuf-compiler + + # Some of our rust modules use FFI and need those to be checked + - name: Get postgres headers + run: make postgres-headers -j$(nproc) + + - name: Produce the build stats + run: cargo build --all --release --timings + + - name: Upload the build stats + uses: actions/upload-artifact@v3 + with: + name: neon-${{ runner.os }}-release-build-stats + path: ./target/cargo-timings/ diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 928a10e555..7ee14a8f41 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -4,7 +4,7 @@ # version, we can consider updating. # See https://tracker.debian.org/pkg/rustc for more details on Debian rustc package, # we use "unstable" version number as the highest version used in the project by default. -channel = "1.62.1" # do update GitHub CI cache values for rust builds, when changing this value +channel = "1.62.1" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html