diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 39a30d9a39..edc456d611 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -8,6 +8,7 @@ self-hosted-runner: - small-arm64 - us-east-2 config-variables: + - AWS_ECR_REGION - AZURE_DEV_CLIENT_ID - AZURE_DEV_REGISTRY_NAME - AZURE_DEV_SUBSCRIPTION_ID @@ -15,23 +16,25 @@ config-variables: - AZURE_PROD_REGISTRY_NAME - AZURE_PROD_SUBSCRIPTION_ID - AZURE_TENANT_ID + - BENCHMARK_INGEST_TARGET_PROJECTID + - BENCHMARK_LARGE_OLTP_PROJECTID - BENCHMARK_PROJECT_ID_PUB - BENCHMARK_PROJECT_ID_SUB - - REMOTE_STORAGE_AZURE_CONTAINER - - REMOTE_STORAGE_AZURE_REGION - - SLACK_UPCOMING_RELEASE_CHANNEL_ID - DEV_AWS_OIDC_ROLE_ARN - - BENCHMARK_INGEST_TARGET_PROJECTID - - PGREGRESS_PG16_PROJECT_ID - - PGREGRESS_PG17_PROJECT_ID - - SLACK_ON_CALL_QA_STAGING_STREAM - DEV_AWS_OIDC_ROLE_MANAGE_BENCHMARK_EC2_VMS_ARN - - SLACK_ON_CALL_STORAGE_STAGING_STREAM - - SLACK_CICD_CHANNEL_ID - - SLACK_STORAGE_CHANNEL_ID + - HETZNER_CACHE_BUCKET + - HETZNER_CACHE_ENDPOINT + - HETZNER_CACHE_REGION - NEON_DEV_AWS_ACCOUNT_ID - NEON_PROD_AWS_ACCOUNT_ID - - AWS_ECR_REGION - - BENCHMARK_LARGE_OLTP_PROJECTID + - PGREGRESS_PG16_PROJECT_ID + - PGREGRESS_PG17_PROJECT_ID + - REMOTE_STORAGE_AZURE_CONTAINER + - REMOTE_STORAGE_AZURE_REGION + - SLACK_CICD_CHANNEL_ID - SLACK_ON_CALL_DEVPROD_STREAM + - SLACK_ON_CALL_QA_STAGING_STREAM + - SLACK_ON_CALL_STORAGE_STAGING_STREAM - SLACK_RUST_CHANNEL_ID + - SLACK_STORAGE_CHANNEL_ID + - SLACK_UPCOMING_RELEASE_CHANNEL_ID diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index 17a5189ccf..8b1314f95b 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -128,29 +128,49 @@ jobs: - name: Cache postgres v14 build id: cache_pg_14 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v14 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} - name: Cache postgres v15 build id: cache_pg_15 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v15 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} - name: Cache postgres v16 build id: cache_pg_16 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v16 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} - name: Cache postgres v17 build id: cache_pg_17 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v17 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }} diff --git a/.github/workflows/_check-codestyle-python.yml b/.github/workflows/_check-codestyle-python.yml index bcd316e0a7..2def92ffa2 100644 --- a/.github/workflows/_check-codestyle-python.yml +++ b/.github/workflows/_check-codestyle-python.yml @@ -37,8 +37,14 @@ jobs: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + - name: Cache poetry deps + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: ~/.cache/pypoetry/virtualenvs key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }} diff --git a/.github/workflows/_check-codestyle-rust.yml b/.github/workflows/_check-codestyle-rust.yml index 4958dfa6c2..4f844b0bf6 100644 --- a/.github/workflows/_check-codestyle-rust.yml +++ b/.github/workflows/_check-codestyle-rust.yml @@ -48,8 +48,13 @@ jobs: submodules: true - name: Cache cargo deps - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: | ~/.cargo/registry !~/.cargo/registry/src diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 5b99efaaf8..148c1ef5af 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -63,8 +63,13 @@ jobs: - name: Cache postgres ${{ matrix.postgres-version }} build id: cache_pg - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/${{ matrix.postgres-version }} key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ matrix.postgres-version }}-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} @@ -129,15 +134,25 @@ jobs: - name: Cache postgres v17 build id: cache_pg - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v17 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache walproposer-lib id: cache_walproposer_lib - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/build/walproposer-lib key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} @@ -203,32 +218,57 @@ jobs: - name: Cache postgres v14 build id: cache_pg - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v14 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v14-${{ steps.pg_rev_v14.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache postgres v15 build id: cache_pg_v15 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v15 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v15-${{ steps.pg_rev_v15.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache postgres v16 build id: cache_pg_v16 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v16 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v16-${{ steps.pg_rev_v16.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache postgres v17 build id: cache_pg_v17 - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/v17 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache cargo deps (only for v17) - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: | ~/.cargo/registry !~/.cargo/registry/src @@ -238,8 +278,13 @@ jobs: - name: Cache walproposer-lib id: cache_walproposer_lib - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: pg_install/build/walproposer-lib key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1e8b79b27d..9ae4fc9d43 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -248,8 +248,13 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Cache poetry deps - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 + uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 with: + endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} + bucket: ${{ vars.HETZNER_CACHE_BUCKET }} + accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} + secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} + use-fallback: false path: ~/.cache/pypoetry/virtualenvs key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }}