mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 21:42:56 +00:00
1d5d168626
## Problem Occasionally getting data from GH cache could be slow, with less than 10MB/s and taking 5+ minutes to download cache: ``` Received 20971520 of 2987085791 (0.7%), 9.9 MBs/sec Received 50331648 of 2987085791 (1.7%), 15.9 MBs/sec ... Received 1065353216 of 2987085791 (35.7%), 4.8 MBs/sec Received 1065353216 of 2987085791 (35.7%), 4.7 MBs/sec ... ``` https://github.com/neondatabase/neon/actions/runs/13956437454/job/39068664599#step:7:17 Resulting in getting cache even longer that build time. ## Summary of changes Switch to the caches, that are closer to the runners, and they provided stable throughput about 70-80MB/s
305 lines
13 KiB
YAML
305 lines
13 KiB
YAML
name: Check neon with MacOS builds
|
|
|
|
on:
|
|
workflow_call:
|
|
inputs:
|
|
pg_versions:
|
|
description: "Array of the pg versions to build for, for example: ['v14', 'v17']"
|
|
type: string
|
|
default: '[]'
|
|
required: false
|
|
rebuild_rust_code:
|
|
description: "Rebuild Rust code"
|
|
type: boolean
|
|
default: false
|
|
required: false
|
|
rebuild_everything:
|
|
description: "If true, rebuild for all versions"
|
|
type: boolean
|
|
default: false
|
|
required: false
|
|
|
|
env:
|
|
RUST_BACKTRACE: 1
|
|
COPT: '-Werror'
|
|
|
|
# TODO: move `check-*` and `files-changed` jobs to the "Caller" Workflow
|
|
# We should care about that as Github has limitations:
|
|
# - You can connect up to four levels of workflows
|
|
# - You can call a maximum of 20 unique reusable workflows from a single workflow file.
|
|
# https://docs.github.com/en/actions/sharing-automations/reusing-workflows#limitations
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
build-pgxn:
|
|
if: |
|
|
(inputs.pg_versions != '[]' || inputs.rebuild_everything) && (
|
|
contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') ||
|
|
github.ref_name == 'main'
|
|
)
|
|
timeout-minutes: 30
|
|
runs-on: macos-15
|
|
strategy:
|
|
matrix:
|
|
postgres-version: ${{ inputs.rebuild_everything && fromJSON('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }}
|
|
env:
|
|
# Use release build only, to have less debug info around
|
|
# Hence keeping target/ (and general cache size) smaller
|
|
BUILD_TYPE: release
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Checkout main repo
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Set pg ${{ matrix.postgres-version }} for caching
|
|
id: pg_rev
|
|
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-${{ matrix.postgres-version }}) | tee -a "${GITHUB_OUTPUT}"
|
|
|
|
- name: Cache postgres ${{ matrix.postgres-version }} build
|
|
id: cache_pg
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/${{ matrix.postgres-version }}
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ matrix.postgres-version }}-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
|
|
- name: Checkout submodule vendor/postgres-${{ matrix.postgres-version }}
|
|
if: steps.cache_pg.outputs.cache-hit != 'true'
|
|
run: |
|
|
git submodule init vendor/postgres-${{ matrix.postgres-version }}
|
|
git submodule update --depth 1 --recursive
|
|
|
|
- name: Install build dependencies
|
|
if: steps.cache_pg.outputs.cache-hit != 'true'
|
|
run: |
|
|
brew install flex bison openssl protobuf icu4c
|
|
|
|
- name: Set extra env for macOS
|
|
if: steps.cache_pg.outputs.cache-hit != 'true'
|
|
run: |
|
|
echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
|
|
echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
|
|
|
|
- name: Build Postgres ${{ matrix.postgres-version }}
|
|
if: steps.cache_pg.outputs.cache-hit != 'true'
|
|
run: |
|
|
make postgres-${{ matrix.postgres-version }} -j$(sysctl -n hw.ncpu)
|
|
|
|
- name: Build Neon Pg Ext ${{ matrix.postgres-version }}
|
|
if: steps.cache_pg.outputs.cache-hit != 'true'
|
|
run: |
|
|
make "neon-pg-ext-${{ matrix.postgres-version }}" -j$(sysctl -n hw.ncpu)
|
|
|
|
- name: Get postgres headers ${{ matrix.postgres-version }}
|
|
if: steps.cache_pg.outputs.cache-hit != 'true'
|
|
run: |
|
|
make postgres-headers-${{ matrix.postgres-version }} -j$(sysctl -n hw.ncpu)
|
|
|
|
build-walproposer-lib:
|
|
if: |
|
|
(inputs.pg_versions != '[]' || inputs.rebuild_everything) && (
|
|
contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') ||
|
|
github.ref_name == 'main'
|
|
)
|
|
timeout-minutes: 30
|
|
runs-on: macos-15
|
|
needs: [build-pgxn]
|
|
env:
|
|
# Use release build only, to have less debug info around
|
|
# Hence keeping target/ (and general cache size) smaller
|
|
BUILD_TYPE: release
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Checkout main repo
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Set pg v17 for caching
|
|
id: pg_rev
|
|
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) | tee -a "${GITHUB_OUTPUT}"
|
|
|
|
- name: Cache postgres v17 build
|
|
id: cache_pg
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/v17
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
|
|
- name: Cache walproposer-lib
|
|
id: cache_walproposer_lib
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/build/walproposer-lib
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
|
|
- name: Checkout submodule vendor/postgres-v17
|
|
if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
|
|
run: |
|
|
git submodule init vendor/postgres-v17
|
|
git submodule update --depth 1 --recursive
|
|
|
|
- name: Install build dependencies
|
|
if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
|
|
run: |
|
|
brew install flex bison openssl protobuf icu4c
|
|
|
|
- name: Set extra env for macOS
|
|
if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
|
|
run: |
|
|
echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
|
|
echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
|
|
|
|
- name: Build walproposer-lib (only for v17)
|
|
if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
|
|
run:
|
|
make walproposer-lib -j$(sysctl -n hw.ncpu)
|
|
|
|
cargo-build:
|
|
if: |
|
|
(inputs.pg_versions != '[]' || inputs.rebuild_rust_code || inputs.rebuild_everything) && (
|
|
contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') ||
|
|
contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') ||
|
|
github.ref_name == 'main'
|
|
)
|
|
timeout-minutes: 30
|
|
runs-on: macos-15
|
|
needs: [build-pgxn, build-walproposer-lib]
|
|
env:
|
|
# Use release build only, to have less debug info around
|
|
# Hence keeping target/ (and general cache size) smaller
|
|
BUILD_TYPE: release
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Checkout main repo
|
|
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
with:
|
|
submodules: true
|
|
|
|
- name: Set pg v14 for caching
|
|
id: pg_rev_v14
|
|
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) | tee -a "${GITHUB_OUTPUT}"
|
|
- name: Set pg v15 for caching
|
|
id: pg_rev_v15
|
|
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) | tee -a "${GITHUB_OUTPUT}"
|
|
- name: Set pg v16 for caching
|
|
id: pg_rev_v16
|
|
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) | tee -a "${GITHUB_OUTPUT}"
|
|
- name: Set pg v17 for caching
|
|
id: pg_rev_v17
|
|
run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) | tee -a "${GITHUB_OUTPUT}"
|
|
|
|
- name: Cache postgres v14 build
|
|
id: cache_pg
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/v14
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v14-${{ steps.pg_rev_v14.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
- name: Cache postgres v15 build
|
|
id: cache_pg_v15
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/v15
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v15-${{ steps.pg_rev_v15.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
- name: Cache postgres v16 build
|
|
id: cache_pg_v16
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/v16
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v16-${{ steps.pg_rev_v16.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
- name: Cache postgres v17 build
|
|
id: cache_pg_v17
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/v17
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
|
|
- name: Cache cargo deps (only for v17)
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: |
|
|
~/.cargo/registry
|
|
!~/.cargo/registry/src
|
|
~/.cargo/git
|
|
target
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust
|
|
|
|
- name: Cache walproposer-lib
|
|
id: cache_walproposer_lib
|
|
uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0
|
|
with:
|
|
endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }}
|
|
bucket: ${{ vars.HETZNER_CACHE_BUCKET }}
|
|
accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }}
|
|
secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
|
|
use-fallback: false
|
|
path: pg_install/build/walproposer-lib
|
|
key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
|
|
|
|
- name: Install build dependencies
|
|
run: |
|
|
brew install flex bison openssl protobuf icu4c
|
|
|
|
- name: Set extra env for macOS
|
|
run: |
|
|
echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
|
|
echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
|
|
|
|
- name: Run cargo build (only for v17)
|
|
run: cargo build --all --release -j$(sysctl -n hw.ncpu)
|
|
|
|
- name: Check that no warnings are produced (only for v17)
|
|
run: ./run_clippy.sh
|