diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 7b2c9c2ce3..24c4e776df 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -32,162 +32,14 @@ permissions: contents: read jobs: - build-pgxn: - if: | - inputs.pg_versions != '[]' || inputs.rebuild_everything || - contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || - contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || - github.ref_name == 'main' - timeout-minutes: 30 - runs-on: macos-15 - strategy: - matrix: - postgres-version: ${{ inputs.rebuild_everything && fromJSON('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }} - env: - # Use release build only, to have less debug info around - # Hence keeping target/ (and general cache size) smaller - BUILD_TYPE: release - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 - with: - egress-policy: audit - - - name: Checkout main repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Set pg ${{ matrix.postgres-version }} for caching - id: pg_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-${{ matrix.postgres-version }}) | tee -a "${GITHUB_OUTPUT}" - - - name: Cache postgres ${{ matrix.postgres-version }} build - id: cache_pg - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: pg_install/${{ matrix.postgres-version }} - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ matrix.postgres-version }}-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - - - name: Checkout submodule vendor/postgres-${{ matrix.postgres-version }} - if: steps.cache_pg.outputs.cache-hit != 'true' - run: | - git submodule init vendor/postgres-${{ matrix.postgres-version }} - git submodule update --depth 1 --recursive - - - name: Install build dependencies - if: steps.cache_pg.outputs.cache-hit != 'true' - run: | - brew install flex bison openssl protobuf icu4c - - - name: Set extra env for macOS - if: steps.cache_pg.outputs.cache-hit != 'true' - run: | - echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV - echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV - - - name: Build Postgres ${{ matrix.postgres-version }} - if: steps.cache_pg.outputs.cache-hit != 'true' - run: | - make postgres-${{ matrix.postgres-version }} -j$(sysctl -n hw.ncpu) - - - name: Build Neon Pg Ext ${{ matrix.postgres-version }} - if: steps.cache_pg.outputs.cache-hit != 'true' - run: | - make "neon-pg-ext-${{ matrix.postgres-version }}" -j$(sysctl -n hw.ncpu) - - - name: Upload "pg_install/${{ matrix.postgres-version }}" artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: pg_install--${{ matrix.postgres-version }} - path: pg_install/${{ matrix.postgres-version }} - # The artifact is supposed to be used by the next job in the same workflow, - # so there’s no need to store it for too long. - retention-days: 1 - - build-walproposer-lib: - if: | - contains(inputs.pg_versions, 'v17') || inputs.rebuild_everything || - contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || - contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || - github.ref_name == 'main' - timeout-minutes: 30 - runs-on: macos-15 - needs: [build-pgxn] - env: - # Use release build only, to have less debug info around - # Hence keeping target/ (and general cache size) smaller - BUILD_TYPE: release - steps: - - name: Harden the runner (Audit all outbound calls) - uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0 - with: - egress-policy: audit - - - name: Checkout main repo - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Set pg v17 for caching - id: pg_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) | tee -a "${GITHUB_OUTPUT}" - - - name: Download "pg_install/v17" artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: pg_install--v17 - path: pg_install/v17 - - # `actions/download-artifact` doesn't preserve permissions: - # https://github.com/actions/download-artifact?tab=readme-ov-file#permission-loss - - name: Make pg_install/v*/bin/* executable - run: | - chmod +x pg_install/v*/bin/* - - - name: Cache walproposer-lib - id: cache_walproposer_lib - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 - with: - path: build/walproposer-lib - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - - - name: Checkout submodule vendor/postgres-v17 - if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' - run: | - git submodule init vendor/postgres-v17 - git submodule update --depth 1 --recursive - - - name: Install build dependencies - if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' - run: | - brew install flex bison openssl protobuf icu4c - - - name: Set extra env for macOS - if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' - run: | - echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV - echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV - - - name: Build walproposer-lib (only for v17) - if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' - run: - make walproposer-lib -j$(sysctl -n hw.ncpu) PG_INSTALL_CACHED=1 - - - name: Upload "build/walproposer-lib" artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: build--walproposer-lib - path: build/walproposer-lib - # The artifact is supposed to be used by the next job in the same workflow, - # so there’s no need to store it for too long. - retention-days: 1 - - cargo-build: + make-all: if: | inputs.pg_versions != '[]' || inputs.rebuild_rust_code || inputs.rebuild_everything || contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || github.ref_name == 'main' - timeout-minutes: 30 + timeout-minutes: 60 runs-on: macos-15 - needs: [build-pgxn, build-walproposer-lib] env: # Use release build only, to have less debug info around # Hence keeping target/ (and general cache size) smaller @@ -203,41 +55,51 @@ jobs: with: submodules: true - - name: Download "pg_install/v14" artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: pg_install--v14 - path: pg_install/v14 - - - name: Download "pg_install/v15" artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: pg_install--v15 - path: pg_install/v15 - - - name: Download "pg_install/v16" artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: pg_install--v16 - path: pg_install/v16 - - - name: Download "pg_install/v17" artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: pg_install--v17 - path: pg_install/v17 - - - name: Download "build/walproposer-lib" artifact - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - name: build--walproposer-lib - path: build/walproposer-lib - - # `actions/download-artifact` doesn't preserve permissions: - # https://github.com/actions/download-artifact?tab=readme-ov-file#permission-loss - - name: Make pg_install/v*/bin/* executable + - name: Install build dependencies run: | - chmod +x pg_install/v*/bin/* + brew install flex bison openssl protobuf icu4c + + - name: Set extra env for macOS + run: | + echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV + echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV + + - name: Restore "pg_install/" cache + id: cache_pg + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 + with: + path: pg_install + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-install-v14-${{ hashFiles('Makefile', 'postgres.mk', 'vendor/revisions.json') }} + + - name: Checkout vendor/postgres submodules + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + git submodule init + git submodule update --depth 1 --recursive + + - name: Build Postgres + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + make postgres -j$(sysctl -n hw.ncpu) + + # This isn't strictly necessary, but it makes the cached and non-cached builds more similar, + # When pg_install is restored from cache, there is no 'build/' directory. By removing it + # in a non-cached build too, we enforce that the rest of the steps don't depend on it, + # so that we notice any build caching bugs earlier. + - name: Remove build artifacts + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + rm -rf build + + # Explicitly update the rust toolchain before running 'make'. The parallel make build can + # invoke 'cargo build' more than once in parallel, for different crates. That's OK, 'cargo' + # does its own locking to prevent concurrent builds from stepping on each other's + # toes. However, it will first try to update the toolchain, and that step is not locked the + # same way. To avoid two toolchain updates running in parallel and stepping on each other's + # toes, ensure that the toolchain is up-to-date beforehand. + - name: Update rust toolchain + run: | + rustup update - name: Cache cargo deps uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 @@ -249,17 +111,12 @@ jobs: target key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust - - name: Install build dependencies - run: | - brew install flex bison openssl protobuf icu4c - - - name: Set extra env for macOS - run: | - echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV - echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV - - - name: Run cargo build - run: cargo build --all --release -j$(sysctl -n hw.ncpu) + # Build the neon-specific postgres extensions, and all the Rust bits. + # + # Pass PG_INSTALL_CACHED=1 because PostgreSQL was already built and cached + # separately. + - name: Build all + run: PG_INSTALL_CACHED=1 BUILD_TYPE=release make -j$(sysctl -n hw.ncpu) all - name: Check that no warnings are produced run: ./run_clippy.sh diff --git a/Makefile b/Makefile index 7f8f436a2e..8ebd27f7c5 100644 --- a/Makefile +++ b/Makefile @@ -102,7 +102,7 @@ all: neon postgres-install neon-pg-ext ### Neon Rust bits # -# The 'postgres_ffi' depends on the Postgres headers. +# The 'postgres_ffi' crate depends on the Postgres headers. .PHONY: neon neon: postgres-headers-install walproposer-lib cargo-target-dir +@echo "Compiling Neon"