diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index db1ea464e6..b950187fe1 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -39,7 +39,7 @@ env: jobs: build-neon: - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} permissions: id-token: write # aws-actions/configure-aws-credentials contents: read @@ -318,7 +318,7 @@ jobs: contents: read statuses: write needs: [ build-neon ] - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} container: image: ${{ inputs.build-tools-image }} credentials: diff --git a/.github/workflows/_check-codestyle-rust.yml b/.github/workflows/_check-codestyle-rust.yml index 6d517abe72..83eeb83e45 100644 --- a/.github/workflows/_check-codestyle-rust.yml +++ b/.github/workflows/_check-codestyle-rust.yml @@ -23,8 +23,8 @@ jobs: check-codestyle-rust: strategy: matrix: - arch: ${{ fromJson(inputs.archs) }} - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }} + arch: ${{ fromJSON(inputs.archs) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }} permissions: packages: read diff --git a/.github/workflows/_meta.yml b/.github/workflows/_meta.yml index bb2f9fa5d9..44802f0525 100644 --- a/.github/workflows/_meta.yml +++ b/.github/workflows/_meta.yml @@ -120,7 +120,7 @@ jobs: - name: Get the release PR run ID id: release-pr-run-id - if: ${{ contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), steps.run-kind.outputs.run-kind) }} + if: ${{ contains(fromJSON('["storage-release", "compute-release", "proxy-release"]'), steps.run-kind.outputs.run-kind) }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index ff7db02e42..e080b06cac 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -441,7 +441,7 @@ jobs: strategy: fail-fast: false - matrix: ${{fromJson(needs.generate-matrices.outputs.pgbench-compare-matrix)}} + matrix: ${{fromJSON(needs.generate-matrices.outputs.pgbench-compare-matrix)}} env: TEST_PG_BENCH_DURATIONS_MATRIX: "60m" @@ -483,7 +483,7 @@ jobs: aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Create Neon Project - if: contains(fromJson('["neonvm-captest-new", "neonvm-captest-new-many-tables", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform) + if: contains(fromJSON('["neonvm-captest-new", "neonvm-captest-new-many-tables", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform) id: create-neon-project uses: ./.github/actions/neon-project-create with: @@ -523,7 +523,7 @@ jobs: # without (neonvm-captest-new) # and with (neonvm-captest-new-many-tables) many relations in the database - name: Create many relations before the run - if: contains(fromJson('["neonvm-captest-new-many-tables"]'), matrix.platform) + if: contains(fromJSON('["neonvm-captest-new-many-tables"]'), matrix.platform) uses: ./.github/actions/run-python-test-set with: build_type: ${{ env.BUILD_TYPE }} @@ -753,7 +753,7 @@ jobs: strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }} + matrix: ${{ fromJSON(needs.generate-matrices.outputs.olap-compare-matrix) }} env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install @@ -880,7 +880,7 @@ jobs: strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-matrices.outputs.tpch-compare-matrix) }} + matrix: ${{ fromJSON(needs.generate-matrices.outputs.tpch-compare-matrix) }} env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install @@ -999,7 +999,7 @@ jobs: strategy: fail-fast: false - matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }} + matrix: ${{ fromJSON(needs.generate-matrices.outputs.olap-compare-matrix) }} env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml index 4eae242395..f7c91e7412 100644 --- a/.github/workflows/build-build-tools-image.yml +++ b/.github/workflows/build-build-tools-image.yml @@ -79,10 +79,10 @@ jobs: env: IMAGE_TAG: ${{ steps.set-variables.outputs.image-tag }} EVERYTHING: | - ${{ contains(fromJson(steps.set-variables.outputs.archs), 'x64') && - contains(fromJson(steps.set-variables.outputs.archs), 'arm64') && - contains(fromJson(steps.set-variables.outputs.debians), 'bullseye') && - contains(fromJson(steps.set-variables.outputs.debians), 'bookworm') }} + ${{ contains(fromJSON(steps.set-variables.outputs.archs), 'x64') && + contains(fromJSON(steps.set-variables.outputs.archs), 'arm64') && + contains(fromJSON(steps.set-variables.outputs.debians), 'bullseye') && + contains(fromJSON(steps.set-variables.outputs.debians), 'bookworm') }} run: | if docker manifest inspect ghcr.io/neondatabase/build-tools:${IMAGE_TAG}; then found=true @@ -99,13 +99,13 @@ jobs: strategy: matrix: - arch: ${{ fromJson(needs.check-image.outputs.archs) }} - debian: ${{ fromJson(needs.check-image.outputs.debians) }} + arch: ${{ fromJSON(needs.check-image.outputs.archs) }} + debian: ${{ fromJSON(needs.check-image.outputs.debians) }} permissions: packages: write - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: - uses: actions/checkout@v4 @@ -168,8 +168,8 @@ jobs: - name: Create multi-arch image env: DEFAULT_DEBIAN_VERSION: bookworm - ARCHS: ${{ join(fromJson(needs.check-image.outputs.archs), ' ') }} - DEBIANS: ${{ join(fromJson(needs.check-image.outputs.debians), ' ') }} + ARCHS: ${{ join(fromJSON(needs.check-image.outputs.archs), ' ') }} + DEBIANS: ${{ join(fromJSON(needs.check-image.outputs.debians), ' ') }} EVERYTHING: ${{ needs.check-image.outputs.everything }} IMAGE_TAG: ${{ needs.check-image.outputs.tag }} run: | diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 347a511e98..b24a872152 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -40,7 +40,7 @@ jobs: runs-on: macos-15 strategy: matrix: - postgres-version: ${{ inputs.rebuild_everything && fromJson('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }} + postgres-version: ${{ inputs.rebuild_everything && fromJSON('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }} env: # Use release build only, to have less debug info around # Hence keeping target/ (and general cache size) smaller diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1762cd9644..6e7c1ce0a5 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -77,14 +77,17 @@ jobs: secrets: inherit check-codestyle-python: - needs: [ check-permissions, build-build-tools-image ] + needs: [ meta, check-permissions, build-build-tools-image ] + # No need to run on `main` because we this in the merge queue + if: ${{ needs.meta.outputs.run-kind == 'pr' }} uses: ./.github/workflows/_check-codestyle-python.yml with: build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm secrets: inherit check-codestyle-jsonnet: - needs: [ check-permissions, build-build-tools-image ] + needs: [ meta, check-permissions, build-build-tools-image ] + if: ${{ contains(fromJSON('["pr", "push-main"]'), needs.meta.outputs.run-kind) }} runs-on: [ self-hosted, small ] container: image: ${{ needs.build-build-tools-image.outputs.image }} @@ -156,7 +159,9 @@ jobs: pass_if_unchanged: true check-codestyle-rust: - needs: [ check-permissions, build-build-tools-image ] + needs: [ meta, check-permissions, build-build-tools-image ] + # No need to run on `main` because we this in the merge queue + if: ${{ needs.meta.outputs.run-kind == 'pr' }} uses: ./.github/workflows/_check-codestyle-rust.yml with: build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm @@ -164,8 +169,8 @@ jobs: secrets: inherit check-dependencies-rust: - needs: [ files-changed, build-build-tools-image ] - if: ${{ needs.files-changed.outputs.check-rust-dependencies == 'true' }} + needs: [ meta, files-changed, build-build-tools-image ] + if: ${{ needs.files-changed.outputs.check-rust-dependencies == 'true' && needs.meta.outputs.run-kind == 'pr' }} uses: ./.github/workflows/cargo-deny.yml with: build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm @@ -173,12 +178,13 @@ jobs: build-and-test-locally: needs: [ meta, build-build-tools-image ] + if: ${{ contains(fromJSON('["pr", "push-main"]'), needs.meta.outputs.run-kind) }} strategy: fail-fast: false matrix: arch: [ x64, arm64 ] # Do not build or run tests in debug for release branches - build-type: ${{ fromJson((startsWith(github.ref_name, 'release') && github.event_name == 'push') && '["release"]' || '["debug", "release"]') }} + build-type: ${{ fromJSON((startsWith(github.ref_name, 'release') && github.event_name == 'push') && '["release"]' || '["debug", "release"]') }} include: - build-type: release arch: arm64 @@ -470,14 +476,20 @@ jobs: }) trigger-e2e-tests: - # Depends on jobs that can get skipped + # !failure() && !cancelled() because it depends on jobs that can get skipped if: >- ${{ ( - !github.event.pull_request.draft - || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') - || needs.meta.outputs.run-kind == 'push-main' - ) && !failure() && !cancelled() + ( + needs.meta.outputs.run-kind == 'pr' + && ( + !github.event.pull_request.draft + || contains(github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') + ) + ) + || contains(fromJSON('["push-main", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) + ) + && !failure() && !cancelled() }} needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, meta ] uses: ./.github/workflows/trigger-e2e-tests.yml @@ -492,7 +504,7 @@ jobs: matrix: arch: [ x64, arm64 ] - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} permissions: packages: write @@ -594,7 +606,7 @@ jobs: debian: bookworm arch: [ x64, arm64 ] - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} steps: - uses: actions/checkout@v4 @@ -715,7 +727,7 @@ jobs: vm-compute-node-image-arch: needs: [ check-permissions, meta, compute-node-image ] if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} permissions: contents: read packages: write @@ -815,7 +827,7 @@ jobs: permissions: packages: read - runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }} + runs-on: ${{ fromJSON(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'small-arm64' || 'small')) }} steps: - uses: actions/checkout@v4 @@ -929,7 +941,7 @@ jobs: env: SOURCE_TAG: >- ${{ - contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), needs.meta.outputs.run-kind) + contains(fromJSON('["storage-release", "compute-release", "proxy-release"]'), needs.meta.outputs.run-kind) && needs.meta.outputs.release-pr-run-id || needs.meta.outputs.build-tag }} @@ -1434,10 +1446,10 @@ jobs: if: | contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - || (needs.check-dependencies-rust.result == 'skipped' && needs.files-changed.outputs.check-rust-dependencies == 'true') - || needs.build-and-test-locally.result == 'skipped' - || needs.check-codestyle-python.result == 'skipped' - || needs.check-codestyle-rust.result == 'skipped' + || (needs.check-dependencies-rust.result == 'skipped' && needs.files-changed.outputs.check-rust-dependencies == 'true' && needs.meta.outputs.run-kind == 'pr') + || (needs.build-and-test-locally.result == 'skipped' && needs.meta.outputs.run-kind == 'pr') + || (needs.check-codestyle-python.result == 'skipped' && needs.meta.outputs.run-kind == 'pr') + || (needs.check-codestyle-rust.result == 'skipped' && needs.meta.outputs.run-kind == 'pr') || needs.files-changed.result == 'skipped' || (needs.push-compute-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)) || (needs.push-neon-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)) diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 90318747b3..74e9a5a9ab 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -71,8 +71,8 @@ jobs: uses: ./.github/workflows/build-macos.yml with: pg_versions: ${{ needs.files-changed.outputs.postgres_changes }} - rebuild_rust_code: ${{ fromJson(needs.files-changed.outputs.rebuild_rust_code) }} - rebuild_everything: ${{ fromJson(needs.files-changed.outputs.rebuild_everything) }} + rebuild_rust_code: ${{ fromJSON(needs.files-changed.outputs.rebuild_rust_code) }} + rebuild_everything: ${{ fromJSON(needs.files-changed.outputs.rebuild_everything) }} gather-rust-build-stats: needs: [ check-permissions, build-build-tools-image, files-changed ] diff --git a/.github/workflows/pre-merge-checks.yml b/.github/workflows/pre-merge-checks.yml index 85b131bb11..bbe4638235 100644 --- a/.github/workflows/pre-merge-checks.yml +++ b/.github/workflows/pre-merge-checks.yml @@ -159,7 +159,7 @@ jobs: ${{ always() && github.event_name == 'merge_group' - && contains(fromJson('["release", "release-proxy", "release-compute"]'), needs.meta.outputs.branch) + && contains(fromJSON('["release", "release-proxy", "release-compute"]'), needs.meta.outputs.branch) }} env: GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}