From 2b6bc73ca187f51ac3e82e56ecb56b76bc63460f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Christian=20Gr=C3=BCnhage?= Date: Thu, 13 Mar 2025 11:35:20 +0100 Subject: [PATCH] Revert "Reuse artifacts from release PRs (#11061)" This commit reverts 803e6f908a31343cd72f33b3019fbcdd986ad1d8, afc9524bc7b6c6edbe22d98f780260ce90ef0b90, 507353404c19f14c867db7341c94b06a954d6845, 48be4df3f3a12c4d6ba049718e35d8a472a8de05 and ef0d4a48a8546625ba9824c86839e71851b9bbdc. --- .github/PULL_REQUEST_TEMPLATE/release-pr.md | 21 ++++ .github/scripts/generate_image_maps.py | 43 ++++---- .github/scripts/lint-release-pr.sh | 110 -------------------- .github/workflows/_create-release-pr.yml | 47 +++------ .github/workflows/_meta.yml | 14 --- .github/workflows/build_and_test.yml | 80 ++++++++------ .github/workflows/fast-forward.yml | 36 ------- .github/workflows/lint-release-pr.yml | 24 ----- .github/workflows/pre-merge-checks.yml | 47 +++------ .github/workflows/release.yml | 6 +- 10 files changed, 125 insertions(+), 303 deletions(-) create mode 100644 .github/PULL_REQUEST_TEMPLATE/release-pr.md delete mode 100755 .github/scripts/lint-release-pr.sh delete mode 100644 .github/workflows/fast-forward.yml delete mode 100644 .github/workflows/lint-release-pr.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/release-pr.md b/.github/PULL_REQUEST_TEMPLATE/release-pr.md new file mode 100644 index 0000000000..44b3094c24 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/release-pr.md @@ -0,0 +1,21 @@ +## Release 202Y-MM-DD + +**NB: this PR must be merged only by 'Create a merge commit'!** + +### Checklist when preparing for release +- [ ] Read or refresh [the release flow guide](https://www.notion.so/neondatabase/Release-general-flow-61f2e39fd45d4d14a70c7749604bd70b) +- [ ] Ask in the [cloud Slack channel](https://neondb.slack.com/archives/C033A2WE6BZ) that you are going to rollout the release. Any blockers? +- [ ] Does this release contain any db migrations? Destructive ones? What is the rollback plan? + + + +### Checklist after release +- [ ] Make sure instructions from PRs included in this release and labeled `manual_release_instructions` are executed (either by you or by people who wrote them). +- [ ] Based on the merged commits write release notes and open a PR into `website` repo ([example](https://github.com/neondatabase/website/pull/219/files)) +- [ ] Check [#dev-production-stream](https://neondb.slack.com/archives/C03F5SM1N02) Slack channel +- [ ] Check [stuck projects page](https://console.neon.tech/admin/projects?sort=last_active&order=desc&stuck=true) +- [ ] Check [recent operation failures](https://console.neon.tech/admin/operations?action=create_timeline%2Cstart_compute%2Cstop_compute%2Csuspend_compute%2Capply_config%2Cdelete_timeline%2Cdelete_tenant%2Ccreate_branch%2Ccheck_availability&sort=updated_at&order=desc&had_retries=some) +- [ ] Check [cloud SLO dashboard](https://neonprod.grafana.net/d/_oWcBMJ7k/cloud-slos?orgId=1) +- [ ] Check [compute startup metrics dashboard](https://neonprod.grafana.net/d/5OkYJEmVz/compute-startup-time) + + diff --git a/.github/scripts/generate_image_maps.py b/.github/scripts/generate_image_maps.py index f67e07024c..39ece5b38f 100644 --- a/.github/scripts/generate_image_maps.py +++ b/.github/scripts/generate_image_maps.py @@ -1,16 +1,14 @@ import itertools import json import os -import sys -source_tag = os.getenv("SOURCE_TAG") -target_tag = os.getenv("TARGET_TAG") -branch = os.getenv("BRANCH") -dev_acr = os.getenv("DEV_ACR") -prod_acr = os.getenv("PROD_ACR") -dev_aws = os.getenv("DEV_AWS") -prod_aws = os.getenv("PROD_AWS") -aws_region = os.getenv("AWS_REGION") +build_tag = os.environ["BUILD_TAG"] +branch = os.environ["BRANCH"] +dev_acr = os.environ["DEV_ACR"] +prod_acr = os.environ["PROD_ACR"] +dev_aws = os.environ["DEV_AWS"] +prod_aws = os.environ["PROD_AWS"] +aws_region = os.environ["AWS_REGION"] components = { "neon": ["neon"], @@ -41,23 +39,24 @@ registries = { outputs: dict[str, dict[str, list[str]]] = {} -target_tags = [target_tag, "latest"] if branch == "main" else [target_tag] -target_stages = ( - ["dev", "prod"] if branch in ["release", "release-proxy", "release-compute"] else ["dev"] -) +target_tags = [build_tag, "latest"] if branch == "main" else [build_tag] +target_stages = ["dev", "prod"] if branch.startswith("release") else ["dev"] for component_name, component_images in components.items(): for stage in target_stages: - outputs[f"{component_name}-{stage}"] = { - f"docker.io/neondatabase/{component_image}:{source_tag}": [ - f"{registry}/{component_image}:{tag}" - for registry, tag in itertools.product(registries[stage], target_tags) - if not (registry == "docker.io/neondatabase" and tag == source_tag) + outputs[f"{component_name}-{stage}"] = dict( + [ + ( + f"docker.io/neondatabase/{component_image}:{build_tag}", + [ + f"{combo[0]}/{component_image}:{combo[1]}" + for combo in itertools.product(registries[stage], target_tags) + ], + ) + for component_image in component_images ] - for component_image in component_images - } + ) -with open(os.getenv("GITHUB_OUTPUT", "/dev/null"), "a") as f: +with open(os.environ["GITHUB_OUTPUT"], "a") as f: for key, value in outputs.items(): f.write(f"{key}={json.dumps(value)}\n") - print(f"Image map for {key}:\n{json.dumps(value, indent=2)}\n\n", file=sys.stderr) diff --git a/.github/scripts/lint-release-pr.sh b/.github/scripts/lint-release-pr.sh deleted file mode 100755 index 6dc5b99f0e..0000000000 --- a/.github/scripts/lint-release-pr.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -DOCS_URL="https://docs.neon.build/overview/repositories/neon.html" - -message() { - if [[ -n "${GITHUB_PR_NUMBER:-}" ]]; then - gh pr comment --repo "${GITHUB_REPOSITORY}" "${GITHUB_PR_NUMBER}" --edit-last --body "$1" \ - || gh pr comment --repo "${GITHUB_REPOSITORY}" "${GITHUB_PR_NUMBER}" --body "$1" - fi - echo "$1" -} - -report_error() { - message "❌ $1 - For more details, see the documentation: ${DOCS_URL}" - - exit 1 -} - -case "$RELEASE_BRANCH" in - "release") COMPONENT="Storage" ;; - "release-proxy") COMPONENT="Proxy" ;; - "release-compute") COMPONENT="Compute" ;; - *) - report_error "Unknown release branch: ${RELEASE_BRANCH}" - ;; -esac - - -# Identify main and release branches -MAIN_BRANCH="origin/main" -REMOTE_RELEASE_BRANCH="origin/${RELEASE_BRANCH}" - -# Find merge base -MERGE_BASE=$(git merge-base "${MAIN_BRANCH}" "${REMOTE_RELEASE_BRANCH}") -echo "Merge base of ${MAIN_BRANCH} and ${RELEASE_BRANCH}: ${MERGE_BASE}" - -# Get the HEAD commit (last commit in PR, expected to be the merge commit) -LAST_COMMIT=$(git rev-parse HEAD) - -MERGE_COMMIT_MESSAGE=$(git log -1 --format=%s "${LAST_COMMIT}") -EXPECTED_MESSAGE_REGEX="^$COMPONENT release [0-9]{4}-[0-9]{2}-[0-9]{2}$" - -if ! [[ "${MERGE_COMMIT_MESSAGE}" =~ ${EXPECTED_MESSAGE_REGEX} ]]; then - report_error "Merge commit message does not match expected pattern: ' release YYYY-MM-DD' - Expected component: ${COMPONENT} - Found: '${MERGE_COMMIT_MESSAGE}'" -fi -echo "✅ Merge commit message is correctly formatted: '${MERGE_COMMIT_MESSAGE}'" - -LAST_COMMIT_PARENTS=$(git cat-file -p "${LAST_COMMIT}" | jq -sR '[capture("parent (?[0-9a-f]{40})"; "g") | .parent]') - -if [[ "$(echo "${LAST_COMMIT_PARENTS}" | jq 'length')" -ne 2 ]]; then - report_error "Last commit must be a merge commit with exactly two parents" -fi - -EXPECTED_RELEASE_HEAD=$(git rev-parse "${REMOTE_RELEASE_BRANCH}") -if echo "${LAST_COMMIT_PARENTS}" | jq -e --arg rel "${EXPECTED_RELEASE_HEAD}" 'index($rel) != null' > /dev/null; then - LINEAR_HEAD=$(echo "${LAST_COMMIT_PARENTS}" | jq -r '[.[] | select(. != $rel)][0]' --arg rel "${EXPECTED_RELEASE_HEAD}") -else - report_error "Last commit must merge the release branch (${RELEASE_BRANCH})" -fi -echo "✅ Last commit correctly merges the previous commit and the release branch" -echo "Top commit of linear history: ${LINEAR_HEAD}" - -MERGE_COMMIT_TREE=$(git rev-parse "${LAST_COMMIT}^{tree}") -LINEAR_HEAD_TREE=$(git rev-parse "${LINEAR_HEAD}^{tree}") - -if [[ "${MERGE_COMMIT_TREE}" != "${LINEAR_HEAD_TREE}" ]]; then - report_error "Tree of merge commit (${MERGE_COMMIT_TREE}) does not match tree of linear history head (${LINEAR_HEAD_TREE}) - This indicates that the merge of ${RELEASE_BRANCH} into this branch was not performed using the merge strategy 'ours'" -fi -echo "✅ Merge commit tree matches the linear history head" - -EXPECTED_PREVIOUS_COMMIT="${LINEAR_HEAD}" - -# Now traverse down the history, ensuring each commit has exactly one parent -CURRENT_COMMIT="${EXPECTED_PREVIOUS_COMMIT}" -while [[ "${CURRENT_COMMIT}" != "${MERGE_BASE}" && "${CURRENT_COMMIT}" != "${EXPECTED_RELEASE_HEAD}" ]]; do - CURRENT_COMMIT_PARENTS=$(git cat-file -p "${CURRENT_COMMIT}" | jq -sR '[capture("parent (?[0-9a-f]{40})"; "g") | .parent]') - - if [[ "$(echo "${CURRENT_COMMIT_PARENTS}" | jq 'length')" -ne 1 ]]; then - report_error "Commit ${CURRENT_COMMIT} must have exactly one parent" - fi - - NEXT_COMMIT=$(echo "${CURRENT_COMMIT_PARENTS}" | jq -r '.[0]') - - if [[ "${NEXT_COMMIT}" == "${MERGE_BASE}" ]]; then - echo "✅ Reached merge base (${MERGE_BASE})" - PR_BASE="${MERGE_BASE}" - elif [[ "${NEXT_COMMIT}" == "${EXPECTED_RELEASE_HEAD}" ]]; then - echo "✅ Reached release branch (${EXPECTED_RELEASE_HEAD})" - PR_BASE="${EXPECTED_RELEASE_HEAD}" - elif [[ -z "${NEXT_COMMIT}" ]]; then - report_error "Unexpected end of commit history before reaching merge base" - fi - - # Move to the next commit in the chain - CURRENT_COMMIT="${NEXT_COMMIT}" -done - -echo "✅ All commits are properly ordered and linear" -echo "✅ Release PR structure is valid" - -echo - -message "Commits that are part of this release: -$(git log --oneline "${PR_BASE}..${LINEAR_HEAD}")" diff --git a/.github/workflows/_create-release-pr.yml b/.github/workflows/_create-release-pr.yml index 9b1d1aa454..3c130c8229 100644 --- a/.github/workflows/_create-release-pr.yml +++ b/.github/workflows/_create-release-pr.yml @@ -7,8 +7,8 @@ on: description: 'Component name' required: true type: string - source-branch: - description: 'Source branch' + release-branch: + description: 'Release branch' required: true type: string secrets: @@ -30,25 +30,17 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.source-branch }} - fetch-depth: 0 + ref: main - name: Set variables id: vars env: COMPONENT_NAME: ${{ inputs.component-name }} - RELEASE_BRANCH: >- - ${{ - false - || inputs.component-name == 'Storage' && 'release' - || inputs.component-name == 'Proxy' && 'release-proxy' - || inputs.component-name == 'Compute' && 'release-compute' - }} + RELEASE_BRANCH: ${{ inputs.release-branch }} run: | today=$(date +'%Y-%m-%d') echo "title=${COMPONENT_NAME} release ${today}" | tee -a ${GITHUB_OUTPUT} echo "rc-branch=rc/${RELEASE_BRANCH}/${today}" | tee -a ${GITHUB_OUTPUT} - echo "release-branch=${RELEASE_BRANCH}" | tee -a ${GITHUB_OUTPUT} - name: Configure git run: | @@ -57,36 +49,31 @@ jobs: - name: Create RC branch env: - RELEASE_BRANCH: ${{ steps.vars.outputs.release-branch }} RC_BRANCH: ${{ steps.vars.outputs.rc-branch }} TITLE: ${{ steps.vars.outputs.title }} run: | - git switch -c "${RC_BRANCH}" + git checkout -b "${RC_BRANCH}" - # Manually create a merge commit on the current branch, keeping the - # tree and setting the parents to the current HEAD and the HEAD of the - # release branch. This commit is what we'll fast-forward the release - # branch to when merging the release branch. - # For details on why, look at - # https://docs.neon.build/overview/repositories/neon.html#background-on-commit-history-of-release-prs - current_tree=$(git rev-parse 'HEAD^{tree}') - release_head=$(git rev-parse "origin/${RELEASE_BRANCH}") - current_head=$(git rev-parse HEAD) - merge_commit=$(git commit-tree -p "${current_head}" -p "${release_head}" -m "${TITLE}" "${current_tree}") - - # Fast-forward the current branch to the newly created merge_commit - git merge --ff-only ${merge_commit} + # create an empty commit to distinguish workflow runs + # from other possible releases from the same commit + git commit --allow-empty -m "${TITLE}" git push origin "${RC_BRANCH}" - - name: Create a PR into ${{ steps.vars.outputs.release-branch }} + - name: Create a PR into ${{ inputs.release-branch }} env: GH_TOKEN: ${{ secrets.ci-access-token }} RC_BRANCH: ${{ steps.vars.outputs.rc-branch }} - RELEASE_BRANCH: ${{ steps.vars.outputs.release-branch }} + RELEASE_BRANCH: ${{ inputs.release-branch }} TITLE: ${{ steps.vars.outputs.title }} run: | + cat << EOF > body.md + ## ${TITLE} + + **Please merge this Pull Request using 'Create a merge commit' button** + EOF + gh pr create --title "${TITLE}" \ - --body "" \ + --body-file "body.md" \ --head "${RC_BRANCH}" \ --base "${RELEASE_BRANCH}" diff --git a/.github/workflows/_meta.yml b/.github/workflows/_meta.yml index c9e7b66efa..cae7fae6a4 100644 --- a/.github/workflows/_meta.yml +++ b/.github/workflows/_meta.yml @@ -21,9 +21,6 @@ on: run-kind: description: "The kind of run we're currently in. Will be one of `push-main`, `storage-release`, `compute-release`, `proxy-release`, `storage-rc-pr`, `compute-rc-pr`, `proxy-rc-pr`, `pr`, or `workflow-dispatch`" value: ${{ jobs.tags.outputs.run-kind }} - release-pr-run-id: - description: "Only available if `run-kind in [storage-release, proxy-release, compute-release]`. Contains the run ID of the `Build and Test` workflow, assuming one with the current commit can be found." - value: ${{ jobs.tags.outputs.release-pr-run-id }} permissions: {} @@ -40,7 +37,6 @@ jobs: proxy: ${{ steps.previous-releases.outputs.proxy }} storage: ${{ steps.previous-releases.outputs.storage }} run-kind: ${{ steps.run-kind.outputs.run-kind }} - release-pr-run-id: ${{ steps.release-pr-run-id.outputs.release-pr-run-id }} permissions: contents: read steps: @@ -117,13 +113,3 @@ jobs: "/repos/${GITHUB_REPOSITORY}/releases" \ | jq -f .github/scripts/previous-releases.jq -r \ | tee -a "${GITHUB_OUTPUT}" - - - name: Get the release PR run ID - id: release-pr-run-id - if: ${{ contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), steps.run-kind.outputs.run-kind) }} - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - run: | - RELEASE_PR_RUN_ID=$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=$CURRENT_SHA" | jq '[.workflow_runs[] | select(.name == "Build and Test") | select(.head_branch | test("^rc/release(-(proxy)|(compute))?/[0-9]{4}-[0-9]{2}-[0-9]{2}$"; "s"))] | first | .id // "Faied to find Build and Test run from RC PR!" | halt_error(1)') - echo "release-pr-run-id=$RELEASE_PR_RUN_ID" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e1ad972a61..1c0971a49d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -476,7 +476,7 @@ jobs: ( !github.event.pull_request.draft || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') - || needs.meta.outputs.run-kind == 'push-main' + || contains(fromJSON('["push-main", "storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind) ) && !failure() && !cancelled() }} needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, meta ] @@ -487,7 +487,7 @@ jobs: neon-image-arch: needs: [ check-permissions, build-build-tools-image, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} strategy: matrix: arch: [ x64, arm64 ] @@ -537,7 +537,7 @@ jobs: neon-image: needs: [ neon-image-arch, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} runs-on: ubuntu-22.04 permissions: id-token: write # aws-actions/configure-aws-credentials @@ -559,7 +559,7 @@ jobs: compute-node-image-arch: needs: [ check-permissions, build-build-tools-image, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} permissions: id-token: write # aws-actions/configure-aws-credentials statuses: write @@ -651,7 +651,7 @@ jobs: compute-node-image: needs: [ compute-node-image-arch, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} permissions: id-token: write # aws-actions/configure-aws-credentials statuses: write @@ -694,7 +694,7 @@ jobs: vm-compute-node-image-arch: needs: [ check-permissions, meta, compute-node-image ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} strategy: fail-fast: false @@ -747,7 +747,7 @@ jobs: vm-compute-node-image: needs: [ vm-compute-node-image-arch, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} runs-on: ubuntu-22.04 strategy: matrix: @@ -773,12 +773,7 @@ jobs: test-images: needs: [ check-permissions, meta, neon-image, compute-node-image ] # Depends on jobs that can get skipped - if: >- - ${{ - !failure() - && !cancelled() - && contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) - }} + if: "!failure() && !cancelled()" strategy: fail-fast: false matrix: @@ -805,7 +800,7 @@ jobs: # Ensure that we don't have bad versions. - name: Verify image versions shell: bash # ensure no set -e for better error messages - if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} run: | pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") @@ -826,19 +821,19 @@ jobs: env: TAG: >- ${{ - needs.meta.outputs.run-kind == 'compute-rc-pr' + contains(fromJSON('["compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) && needs.meta.outputs.previous-storage-release || needs.meta.outputs.build-tag }} COMPUTE_TAG: >- ${{ - contains(fromJSON('["storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) + contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) && needs.meta.outputs.previous-compute-release || needs.meta.outputs.build-tag }} TEST_EXTENSIONS_TAG: >- ${{ - contains(fromJSON('["storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) + contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) && 'latest' || needs.meta.outputs.build-tag }} @@ -890,13 +885,7 @@ jobs: id: generate run: python3 .github/scripts/generate_image_maps.py env: - SOURCE_TAG: >- - ${{ - contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), needs.meta.outputs.run-kind) - && needs.meta.outputs.release-pr-run-id - || needs.meta.outputs.build-tag - }} - TARGET_TAG: ${{ needs.meta.outputs.build-tag }} + BUILD_TAG: "${{ needs.meta.outputs.build-tag }}" BRANCH: "${{ github.ref_name }}" DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}" PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}" @@ -906,7 +895,7 @@ jobs: push-neon-image-dev: needs: [ meta, generate-image-maps, neon-image ] - if: ${{ !failure() && !cancelled() && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} uses: ./.github/workflows/_push-to-container-registry.yml permissions: id-token: write # Required for aws/azure login @@ -924,7 +913,7 @@ jobs: push-compute-image-dev: needs: [ meta, generate-image-maps, vm-compute-node-image ] - if: ${{ !failure() && !cancelled() && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} uses: ./.github/workflows/_push-to-container-registry.yml permissions: id-token: write # Required for aws/azure login @@ -1246,7 +1235,7 @@ jobs: # The job runs on `release` branch and copies compatibility data and Neon artifact from the last *release PR* to the latest directory promote-compatibility-data: - needs: [ meta, deploy ] + needs: [ deploy ] permissions: id-token: write # aws-actions/configure-aws-credentials statuses: write @@ -1256,6 +1245,37 @@ jobs: runs-on: ubuntu-22.04 steps: + - name: Fetch GITHUB_RUN_ID and COMMIT_SHA for the last merged release PR + id: fetch-last-release-pr-info + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + branch_name_and_pr_number=$(gh pr list \ + --repo "${GITHUB_REPOSITORY}" \ + --base release \ + --state merged \ + --limit 10 \ + --json mergeCommit,headRefName,number \ + --jq ".[] | select(.mergeCommit.oid==\"${GITHUB_SHA}\") | { branch_name: .headRefName, pr_number: .number }") + branch_name=$(echo "${branch_name_and_pr_number}" | jq -r '.branch_name') + pr_number=$(echo "${branch_name_and_pr_number}" | jq -r '.pr_number') + + run_id=$(gh run list \ + --repo "${GITHUB_REPOSITORY}" \ + --workflow build_and_test.yml \ + --branch "${branch_name}" \ + --json databaseId \ + --limit 1 \ + --jq '.[].databaseId') + + last_commit_sha=$(gh pr view "${pr_number}" \ + --repo "${GITHUB_REPOSITORY}" \ + --json commits \ + --jq '.commits[-1].oid') + + echo "run-id=${run_id}" | tee -a ${GITHUB_OUTPUT} + echo "commit-sha=${last_commit_sha}" | tee -a ${GITHUB_OUTPUT} + - uses: aws-actions/configure-aws-credentials@v4 with: aws-region: eu-central-1 @@ -1266,8 +1286,8 @@ jobs: env: BUCKET: neon-github-public-dev AWS_REGION: eu-central-1 - COMMIT_SHA: ${{ github.sha }} - RUN_ID: ${{ needs.meta.outputs.release-pr-run-id }} + COMMIT_SHA: ${{ steps.fetch-last-release-pr-info.outputs.commit-sha }} + RUN_ID: ${{ steps.fetch-last-release-pr-info.outputs.run-id }} run: | old_prefix="artifacts/${COMMIT_SHA}/${RUN_ID}" new_prefix="artifacts/latest" @@ -1356,5 +1376,5 @@ jobs: || needs.files-changed.result == 'skipped' || (needs.push-compute-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)) || (needs.push-neon-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)) - || (needs.test-images.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind)) + || needs.test-images.result == 'skipped' || (needs.trigger-custom-extensions-build-and-wait.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)) diff --git a/.github/workflows/fast-forward.yml b/.github/workflows/fast-forward.yml deleted file mode 100644 index bc63ff120d..0000000000 --- a/.github/workflows/fast-forward.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Fast forward merge -on: - pull_request: - types: [labeled] - branches: - - release - - release-proxy - - release-compute - -jobs: - fast-forward: - if: ${{ github.event.label.name == 'fast-forward' }} - runs-on: ubuntu-22.04 - - steps: - - name: Remove fast-forward label to PR - env: - GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} - run: | - gh pr edit ${{ github.event.pull_request.number }} --repo "${GITHUB_REPOSITORY}" --remove-label "fast-forward" - - - name: Fast forwarding - uses: sequoia-pgp/fast-forward@ea7628bedcb0b0b96e94383ada458d812fca4979 - # See https://docs.github.com/en/graphql/reference/enums#mergestatestatus - if: ${{ github.event.pull_request.mergeable_state == 'clean' }} - with: - merge: true - comment: on-error - github_token: ${{ secrets.CI_ACCESS_TOKEN }} - - - name: Comment if mergeable_state is not clean - if: ${{ github.event.pull_request.mergeable_state != 'clean' }} - run: | - gh pr comment ${{ github.event.pull_request.number }} \ - --repo "${GITHUB_REPOSITORY}" \ - --body "Not trying to forward pull-request, because \`mergeable_state\` is \`${{ github.event.pull_request.mergeable_state }}\`, not \`clean\`." diff --git a/.github/workflows/lint-release-pr.yml b/.github/workflows/lint-release-pr.yml deleted file mode 100644 index b7d010f66d..0000000000 --- a/.github/workflows/lint-release-pr.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: Lint Release PR - -on: - pull_request: - branches: - - release - - release-proxy - - release-compute - -jobs: - lint-release-pr: - runs-on: ubuntu-22.04 - steps: - - name: Checkout PR branch - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Fetch full history for git operations - ref: ${{ github.event.pull_request.head.ref }} - - - name: Run lint script - env: - RELEASE_BRANCH: ${{ github.base_ref }} - run: | - ./.github/scripts/lint-release-pr.sh diff --git a/.github/workflows/pre-merge-checks.yml b/.github/workflows/pre-merge-checks.yml index 1e81550314..c47b3fe0de 100644 --- a/.github/workflows/pre-merge-checks.yml +++ b/.github/workflows/pre-merge-checks.yml @@ -8,6 +8,8 @@ on: - .github/workflows/build-build-tools-image.yml - .github/workflows/pre-merge-checks.yml merge_group: + branches: + - main defaults: run: @@ -17,13 +19,11 @@ defaults: permissions: {} jobs: - meta: + get-changed-files: runs-on: ubuntu-22.04 outputs: python-changed: ${{ steps.python-src.outputs.any_changed }} rust-changed: ${{ steps.rust-src.outputs.any_changed }} - branch: ${{ steps.group-metadata.outputs.branch }} - pr-number: ${{ steps.group-metadata.outputs.pr-number }} steps: - uses: actions/checkout@v4 @@ -58,20 +58,12 @@ jobs: echo "${PYTHON_CHANGED_FILES}" echo "${RUST_CHANGED_FILES}" - - name: Merge group metadata - if: ${{ github.event_name == 'merge_group' }} - id: group-metadata - env: - MERGE_QUEUE_REF: ${{ github.event.merge_group.head_ref }} - run: | - echo $MERGE_QUEUE_REF | jq -Rr 'capture("refs/heads/gh-readonly-queue/(?.*)/pr-(?[0-9]+)-[0-9a-f]{40}") | ["branch=" + .branch, "pr-number=" + .pr_number] | .[]' | tee -a "${GITHUB_OUTPUT}" - build-build-tools-image: if: | false - || needs.meta.outputs.python-changed == 'true' - || needs.meta.outputs.rust-changed == 'true' - needs: [ meta ] + || needs.get-changed-files.outputs.python-changed == 'true' + || needs.get-changed-files.outputs.rust-changed == 'true' + needs: [ get-changed-files ] uses: ./.github/workflows/build-build-tools-image.yml with: # Build only one combination to save time @@ -80,8 +72,8 @@ jobs: secrets: inherit check-codestyle-python: - if: needs.meta.outputs.python-changed == 'true' - needs: [ meta, build-build-tools-image ] + if: needs.get-changed-files.outputs.python-changed == 'true' + needs: [ get-changed-files, build-build-tools-image ] uses: ./.github/workflows/_check-codestyle-python.yml with: # `-bookworm-x64` suffix should match the combination in `build-build-tools-image` @@ -89,8 +81,8 @@ jobs: secrets: inherit check-codestyle-rust: - if: needs.meta.outputs.rust-changed == 'true' - needs: [ meta, build-build-tools-image ] + if: needs.get-changed-files.outputs.rust-changed == 'true' + needs: [ get-changed-files, build-build-tools-image ] uses: ./.github/workflows/_check-codestyle-rust.yml with: # `-bookworm-x64` suffix should match the combination in `build-build-tools-image` @@ -109,7 +101,7 @@ jobs: statuses: write # for `github.repos.createCommitStatus(...)` contents: write needs: - - meta + - get-changed-files - check-codestyle-python - check-codestyle-rust runs-on: ubuntu-22.04 @@ -137,20 +129,7 @@ jobs: run: exit 1 if: | false - || (github.event_name == 'merge_group' && needs.meta.outputs.branch != 'main') - || (needs.check-codestyle-python.result == 'skipped' && needs.meta.outputs.python-changed == 'true') - || (needs.check-codestyle-rust.result == 'skipped' && needs.meta.outputs.rust-changed == 'true') + || (needs.check-codestyle-python.result == 'skipped' && needs.get-changed-files.outputs.python-changed == 'true') + || (needs.check-codestyle-rust.result == 'skipped' && needs.get-changed-files.outputs.rust-changed == 'true') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - - - name: Add fast-forward label to PR to trigger fast-forward merge - if: >- - ${{ - always() - && github.event_name == 'merge_group' - && contains(fromJson('["release", "release-proxy", "release-compute"]'), github.base_ref) - }} - env: - GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} - run: >- - gh pr edit ${{ needs.meta.outputs.pr-number }} --repo "${GITHUB_REPOSITORY}" --add-label "fast-forward" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a88ddecd0a..919846ce44 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: uses: ./.github/workflows/_create-release-pr.yml with: component-name: 'Storage' - source-branch: ${{ github.ref_name }} + release-branch: 'release' secrets: ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }} @@ -51,7 +51,7 @@ jobs: uses: ./.github/workflows/_create-release-pr.yml with: component-name: 'Proxy' - source-branch: ${{ github.ref_name }} + release-branch: 'release-proxy' secrets: ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }} @@ -64,6 +64,6 @@ jobs: uses: ./.github/workflows/_create-release-pr.yml with: component-name: 'Compute' - source-branch: ${{ github.ref_name }} + release-branch: 'release-compute' secrets: ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }}