diff --git a/.github/PULL_REQUEST_TEMPLATE/release-pr.md b/.github/PULL_REQUEST_TEMPLATE/release-pr.md deleted file mode 100644 index 44b3094c24..0000000000 --- a/.github/PULL_REQUEST_TEMPLATE/release-pr.md +++ /dev/null @@ -1,21 +0,0 @@ -## Release 202Y-MM-DD - -**NB: this PR must be merged only by 'Create a merge commit'!** - -### Checklist when preparing for release -- [ ] Read or refresh [the release flow guide](https://www.notion.so/neondatabase/Release-general-flow-61f2e39fd45d4d14a70c7749604bd70b) -- [ ] Ask in the [cloud Slack channel](https://neondb.slack.com/archives/C033A2WE6BZ) that you are going to rollout the release. Any blockers? -- [ ] Does this release contain any db migrations? Destructive ones? What is the rollback plan? - - - -### Checklist after release -- [ ] Make sure instructions from PRs included in this release and labeled `manual_release_instructions` are executed (either by you or by people who wrote them). -- [ ] Based on the merged commits write release notes and open a PR into `website` repo ([example](https://github.com/neondatabase/website/pull/219/files)) -- [ ] Check [#dev-production-stream](https://neondb.slack.com/archives/C03F5SM1N02) Slack channel -- [ ] Check [stuck projects page](https://console.neon.tech/admin/projects?sort=last_active&order=desc&stuck=true) -- [ ] Check [recent operation failures](https://console.neon.tech/admin/operations?action=create_timeline%2Cstart_compute%2Cstop_compute%2Csuspend_compute%2Capply_config%2Cdelete_timeline%2Cdelete_tenant%2Ccreate_branch%2Ccheck_availability&sort=updated_at&order=desc&had_retries=some) -- [ ] Check [cloud SLO dashboard](https://neonprod.grafana.net/d/_oWcBMJ7k/cloud-slos?orgId=1) -- [ ] Check [compute startup metrics dashboard](https://neonprod.grafana.net/d/5OkYJEmVz/compute-startup-time) - - diff --git a/.github/scripts/generate_image_maps.py b/.github/scripts/generate_image_maps.py index 39ece5b38f..f67e07024c 100644 --- a/.github/scripts/generate_image_maps.py +++ b/.github/scripts/generate_image_maps.py @@ -1,14 +1,16 @@ import itertools import json import os +import sys -build_tag = os.environ["BUILD_TAG"] -branch = os.environ["BRANCH"] -dev_acr = os.environ["DEV_ACR"] -prod_acr = os.environ["PROD_ACR"] -dev_aws = os.environ["DEV_AWS"] -prod_aws = os.environ["PROD_AWS"] -aws_region = os.environ["AWS_REGION"] +source_tag = os.getenv("SOURCE_TAG") +target_tag = os.getenv("TARGET_TAG") +branch = os.getenv("BRANCH") +dev_acr = os.getenv("DEV_ACR") +prod_acr = os.getenv("PROD_ACR") +dev_aws = os.getenv("DEV_AWS") +prod_aws = os.getenv("PROD_AWS") +aws_region = os.getenv("AWS_REGION") components = { "neon": ["neon"], @@ -39,24 +41,23 @@ registries = { outputs: dict[str, dict[str, list[str]]] = {} -target_tags = [build_tag, "latest"] if branch == "main" else [build_tag] -target_stages = ["dev", "prod"] if branch.startswith("release") else ["dev"] +target_tags = [target_tag, "latest"] if branch == "main" else [target_tag] +target_stages = ( + ["dev", "prod"] if branch in ["release", "release-proxy", "release-compute"] else ["dev"] +) for component_name, component_images in components.items(): for stage in target_stages: - outputs[f"{component_name}-{stage}"] = dict( - [ - ( - f"docker.io/neondatabase/{component_image}:{build_tag}", - [ - f"{combo[0]}/{component_image}:{combo[1]}" - for combo in itertools.product(registries[stage], target_tags) - ], - ) - for component_image in component_images + outputs[f"{component_name}-{stage}"] = { + f"docker.io/neondatabase/{component_image}:{source_tag}": [ + f"{registry}/{component_image}:{tag}" + for registry, tag in itertools.product(registries[stage], target_tags) + if not (registry == "docker.io/neondatabase" and tag == source_tag) ] - ) + for component_image in component_images + } -with open(os.environ["GITHUB_OUTPUT"], "a") as f: +with open(os.getenv("GITHUB_OUTPUT", "/dev/null"), "a") as f: for key, value in outputs.items(): f.write(f"{key}={json.dumps(value)}\n") + print(f"Image map for {key}:\n{json.dumps(value, indent=2)}\n\n", file=sys.stderr) diff --git a/.github/scripts/lint-release-pr.sh b/.github/scripts/lint-release-pr.sh new file mode 100755 index 0000000000..8e081000f9 --- /dev/null +++ b/.github/scripts/lint-release-pr.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +set -euo pipefail + +DOCS_URL="https://docs.neon.build/overview/repositories/neon.html" + +message() { + if [[ -n "${GITHUB_PR_NUMBER:-}" ]]; then + gh pr comment --repo "${GITHUB_REPOSITORY}" "${GITHUB_PR_NUMBER}" --edit-last --body "$1" \ + || gh pr comment --repo "${GITHUB_REPOSITORY}" "${GITHUB_PR_NUMBER}" --body "$1" + fi + echo "$1" +} + +report_error() { + message "❌ $1 + For more details, see the documentation: ${DOCS_URL}" + + exit 1 +} + +case "$RELEASE_BRANCH" in + "release") COMPONENT="Storage" ;; + "release-proxy") COMPONENT="Proxy" ;; + "release-compute") COMPONENT="Compute" ;; + *) + report_error "Unknown release branch: ${RELEASE_BRANCH}" + ;; +esac + + +# Identify main and release branches +MAIN_BRANCH="origin/main" +REMOTE_RELEASE_BRANCH="origin/${RELEASE_BRANCH}" + +# Find merge base +MERGE_BASE=$(git merge-base "${MAIN_BRANCH}" "${REMOTE_RELEASE_BRANCH}") +echo "Merge base of ${MAIN_BRANCH} and ${RELEASE_BRANCH}: ${MERGE_BASE}" + +# Get the HEAD commit (last commit in PR, expected to be the merge commit) +LAST_COMMIT=$(git rev-parse HEAD) + +MERGE_COMMIT_MESSAGE=$(git log -1 --format=%s "${LAST_COMMIT}") +EXPECTED_MESSAGE_REGEX="^$COMPONENT release [0-9]{4}-[0-9]{2}-[0-9]{2}$" + +if ! [[ "${MERGE_COMMIT_MESSAGE}" =~ ${EXPECTED_MESSAGE_REGEX} ]]; then + report_error "Merge commit message does not match expected pattern: ' release YYYY-MM-DD' + Expected component: ${COMPONENT} + Found: '${MERGE_COMMIT_MESSAGE}'" +fi +echo "✅ Merge commit message is correctly formatted: '${MERGE_COMMIT_MESSAGE}'" + +LAST_COMMIT_PARENTS=$(git cat-file -p "${LAST_COMMIT}" | jq -sR '[capture("parent (?[0-9a-f]{40})"; "g") | .parent]') + +if [[ "$(echo "${LAST_COMMIT_PARENTS}" | jq 'length')" -ne 2 ]]; then + report_error "Last commit must be a merge commit with exactly two parents" +fi + +EXPECTED_RELEASE_HEAD=$(git rev-parse "${REMOTE_RELEASE_BRANCH}") +if echo "${LAST_COMMIT_PARENTS}" | jq -e --arg rel "${EXPECTED_RELEASE_HEAD}" 'index($rel) != null' > /dev/null; then + LINEAR_HEAD=$(echo "${LAST_COMMIT_PARENTS}" | jq -r '[.[] | select(. != $rel)][0]' --arg rel "${EXPECTED_RELEASE_HEAD}") +else + report_error "Last commit must merge the release branch (${RELEASE_BRANCH})" +fi +echo "✅ Last commit correctly merges the previous commit and the release branch" +echo "Top commit of linear history: ${LINEAR_HEAD}" + +MERGE_COMMIT_TREE=$(git rev-parse "${LAST_COMMIT}^{tree}") +LINEAR_HEAD_TREE=$(git rev-parse "${LINEAR_HEAD}^{tree}") + +if [[ "${MERGE_COMMIT_TREE}" != "${LINEAR_HEAD_TREE}" ]]; then + report_error "Tree of merge commit (${MERGE_COMMIT_TREE}) does not match tree of linear history head (${LINEAR_HEAD_TREE}) + This indicates that the merge of ${RELEASE_BRANCH} into this branch was not performed using the merge strategy 'ours'" +fi +echo "✅ Merge commit tree matches the linear history head" + +EXPECTED_PREVIOUS_COMMIT="${LINEAR_HEAD}" + +# Now traverse down the history, ensuring each commit has exactly one parent +CURRENT_COMMIT="${EXPECTED_PREVIOUS_COMMIT}" +while [[ "${CURRENT_COMMIT}" != "${MERGE_BASE}" && "${CURRENT_COMMIT}" != "${EXPECTED_RELEASE_HEAD}" ]]; do + CURRENT_COMMIT_PARENTS=$(git cat-file -p "${CURRENT_COMMIT}" | jq -sR '[capture("parent (?[0-9a-f]{40})"; "g") | .parent]') + + if [[ "$(echo "${CURRENT_COMMIT_PARENTS}" | jq 'length')" -ne 1 ]]; then + report_error "Commit ${CURRENT_COMMIT} must have exactly one parent" + fi + + NEXT_COMMIT=$(echo "${CURRENT_COMMIT_PARENTS}" | jq -r '.[0]') + + if [[ "${NEXT_COMMIT}" == "${MERGE_BASE}" ]]; then + echo "✅ Reached merge base (${MERGE_BASE})" + PR_BASE="${MERGE_BASE}" + if [[ "${NEXT_COMMIT}" == "${EXPECTED_RELEASE_HEAD}" ]]; then + echo "✅ Reached release branch (${EXPECTED_RELEASE_HEAD})" + PR_BASE="${EXPECTED_RELEASE_HEAD}" + elif [[ -z "${NEXT_COMMIT}" ]]; then + report_error "Unexpected end of commit history before reaching merge base" + fi + + # Move to the next commit in the chain + CURRENT_COMMIT="${NEXT_COMMIT}" +done + +echo "✅ All commits are properly ordered and linear" +echo "✅ Release PR structure is valid" + +echo + +message "Commits that are part of this release: +$(git log --oneline "${PR_BASE}..${LINEAR_HEAD}")" diff --git a/.github/workflows/_create-release-pr.yml b/.github/workflows/_create-release-pr.yml index 3c130c8229..82acbc0f84 100644 --- a/.github/workflows/_create-release-pr.yml +++ b/.github/workflows/_create-release-pr.yml @@ -7,8 +7,8 @@ on: description: 'Component name' required: true type: string - release-branch: - description: 'Release branch' + source-branch: + description: 'Source branch' required: true type: string secrets: @@ -30,17 +30,24 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: main + ref: ${{ inputs.source-branch }} - name: Set variables id: vars env: COMPONENT_NAME: ${{ inputs.component-name }} - RELEASE_BRANCH: ${{ inputs.release-branch }} + RELEASE_BRANCH: >- + ${{ + false + || inputs.component-name == 'Storage' && 'release' + || inputs.component-name == 'Proxy' && 'release-proxy' + || inputs.component-name == 'Compute' && 'release-compute' + }} run: | today=$(date +'%Y-%m-%d') echo "title=${COMPONENT_NAME} release ${today}" | tee -a ${GITHUB_OUTPUT} echo "rc-branch=rc/${RELEASE_BRANCH}/${today}" | tee -a ${GITHUB_OUTPUT} + echo "release-branch=${RELEASE_BRANCH}" | tee -a ${GITHUB_OUTPUT} - name: Configure git run: | @@ -49,31 +56,35 @@ jobs: - name: Create RC branch env: + RELEASE_BRANCH: ${{ steps.vars.outputs.release-branch }} RC_BRANCH: ${{ steps.vars.outputs.rc-branch }} TITLE: ${{ steps.vars.outputs.title }} run: | - git checkout -b "${RC_BRANCH}" + git switch -c "${RC_BRANCH}" - # create an empty commit to distinguish workflow runs - # from other possible releases from the same commit - git commit --allow-empty -m "${TITLE}" + # Manually create a merge commit on the current branch, keeping the + # tree and setting the parents to the current HEAD and the HEAD of the + # release branch. This commit is what we'll fast-forward the release + # branch to when merging the release branch. + # For details on why, look at + # https://docs.neon.build/overview/repositories/neon.html#background-on-commit-history-of-release-prs + current_tree=$(git rev-parse 'HEAD^{tree}') + release_head=$(git rev-parse "${RELEASE_BRANCH}") + current_head=$(git rev-parse HEAD) + merge_commit=$(git commit-tree -p "${current_head}" -p "${release_head}" -m "${TITLE}" "${current_tree}") + + # Fast-forward the current branch to the newly created merge_commit + git merge --ff-only ${merge_commit} git push origin "${RC_BRANCH}" - - name: Create a PR into ${{ inputs.release-branch }} + - name: Create a PR into ${{ steps.vars.outputs.release-branch }} env: GH_TOKEN: ${{ secrets.ci-access-token }} RC_BRANCH: ${{ steps.vars.outputs.rc-branch }} - RELEASE_BRANCH: ${{ inputs.release-branch }} + RELEASE_BRANCH: ${{ steps.vars.outputs.release-branch }} TITLE: ${{ steps.vars.outputs.title }} run: | - cat << EOF > body.md - ## ${TITLE} - - **Please merge this Pull Request using 'Create a merge commit' button** - EOF - gh pr create --title "${TITLE}" \ - --body-file "body.md" \ --head "${RC_BRANCH}" \ --base "${RELEASE_BRANCH}" diff --git a/.github/workflows/_meta.yml b/.github/workflows/_meta.yml index cae7fae6a4..c9e7b66efa 100644 --- a/.github/workflows/_meta.yml +++ b/.github/workflows/_meta.yml @@ -21,6 +21,9 @@ on: run-kind: description: "The kind of run we're currently in. Will be one of `push-main`, `storage-release`, `compute-release`, `proxy-release`, `storage-rc-pr`, `compute-rc-pr`, `proxy-rc-pr`, `pr`, or `workflow-dispatch`" value: ${{ jobs.tags.outputs.run-kind }} + release-pr-run-id: + description: "Only available if `run-kind in [storage-release, proxy-release, compute-release]`. Contains the run ID of the `Build and Test` workflow, assuming one with the current commit can be found." + value: ${{ jobs.tags.outputs.release-pr-run-id }} permissions: {} @@ -37,6 +40,7 @@ jobs: proxy: ${{ steps.previous-releases.outputs.proxy }} storage: ${{ steps.previous-releases.outputs.storage }} run-kind: ${{ steps.run-kind.outputs.run-kind }} + release-pr-run-id: ${{ steps.release-pr-run-id.outputs.release-pr-run-id }} permissions: contents: read steps: @@ -113,3 +117,13 @@ jobs: "/repos/${GITHUB_REPOSITORY}/releases" \ | jq -f .github/scripts/previous-releases.jq -r \ | tee -a "${GITHUB_OUTPUT}" + + - name: Get the release PR run ID + id: release-pr-run-id + if: ${{ contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), steps.run-kind.outputs.run-kind) }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + RELEASE_PR_RUN_ID=$(gh api "/repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=$CURRENT_SHA" | jq '[.workflow_runs[] | select(.name == "Build and Test") | select(.head_branch | test("^rc/release(-(proxy)|(compute))?/[0-9]{4}-[0-9]{2}-[0-9]{2}$"; "s"))] | first | .id // "Faied to find Build and Test run from RC PR!" | halt_error(1)') + echo "release-pr-run-id=$RELEASE_PR_RUN_ID" | tee -a $GITHUB_OUTPUT diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1c0971a49d..e1ad972a61 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -476,7 +476,7 @@ jobs: ( !github.event.pull_request.draft || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') - || contains(fromJSON('["push-main", "storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind) + || needs.meta.outputs.run-kind == 'push-main' ) && !failure() && !cancelled() }} needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, meta ] @@ -487,7 +487,7 @@ jobs: neon-image-arch: needs: [ check-permissions, build-build-tools-image, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} strategy: matrix: arch: [ x64, arm64 ] @@ -537,7 +537,7 @@ jobs: neon-image: needs: [ neon-image-arch, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} runs-on: ubuntu-22.04 permissions: id-token: write # aws-actions/configure-aws-credentials @@ -559,7 +559,7 @@ jobs: compute-node-image-arch: needs: [ check-permissions, build-build-tools-image, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} permissions: id-token: write # aws-actions/configure-aws-credentials statuses: write @@ -651,7 +651,7 @@ jobs: compute-node-image: needs: [ compute-node-image-arch, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} permissions: id-token: write # aws-actions/configure-aws-credentials statuses: write @@ -694,7 +694,7 @@ jobs: vm-compute-node-image-arch: needs: [ check-permissions, meta, compute-node-image ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', matrix.arch == 'arm64' && 'large-arm64' || 'large')) }} strategy: fail-fast: false @@ -747,7 +747,7 @@ jobs: vm-compute-node-image: needs: [ vm-compute-node-image-arch, meta ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} runs-on: ubuntu-22.04 strategy: matrix: @@ -773,7 +773,12 @@ jobs: test-images: needs: [ check-permissions, meta, neon-image, compute-node-image ] # Depends on jobs that can get skipped - if: "!failure() && !cancelled()" + if: >- + ${{ + !failure() + && !cancelled() + && contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) + }} strategy: fail-fast: false matrix: @@ -800,7 +805,7 @@ jobs: # Ensure that we don't have bad versions. - name: Verify image versions shell: bash # ensure no set -e for better error messages - if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} run: | pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") @@ -821,19 +826,19 @@ jobs: env: TAG: >- ${{ - contains(fromJSON('["compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) + needs.meta.outputs.run-kind == 'compute-rc-pr' && needs.meta.outputs.previous-storage-release || needs.meta.outputs.build-tag }} COMPUTE_TAG: >- ${{ - contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) + contains(fromJSON('["storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) && needs.meta.outputs.previous-compute-release || needs.meta.outputs.build-tag }} TEST_EXTENSIONS_TAG: >- ${{ - contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) + contains(fromJSON('["storage-rc-pr", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) && 'latest' || needs.meta.outputs.build-tag }} @@ -885,7 +890,13 @@ jobs: id: generate run: python3 .github/scripts/generate_image_maps.py env: - BUILD_TAG: "${{ needs.meta.outputs.build-tag }}" + SOURCE_TAG: >- + ${{ + contains(fromJson('["storage-release", "compute-release", "proxy-release"]'), needs.meta.outputs.run-kind) + && needs.meta.outputs.release-pr-run-id + || needs.meta.outputs.build-tag + }} + TARGET_TAG: ${{ needs.meta.outputs.build-tag }} BRANCH: "${{ github.ref_name }}" DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}" PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}" @@ -895,7 +906,7 @@ jobs: push-neon-image-dev: needs: [ meta, generate-image-maps, neon-image ] - if: ${{ contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ !failure() && !cancelled() && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }} uses: ./.github/workflows/_push-to-container-registry.yml permissions: id-token: write # Required for aws/azure login @@ -913,7 +924,7 @@ jobs: push-compute-image-dev: needs: [ meta, generate-image-maps, vm-compute-node-image ] - if: ${{ contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} + if: ${{ !failure() && !cancelled() && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }} uses: ./.github/workflows/_push-to-container-registry.yml permissions: id-token: write # Required for aws/azure login @@ -1235,7 +1246,7 @@ jobs: # The job runs on `release` branch and copies compatibility data and Neon artifact from the last *release PR* to the latest directory promote-compatibility-data: - needs: [ deploy ] + needs: [ meta, deploy ] permissions: id-token: write # aws-actions/configure-aws-credentials statuses: write @@ -1245,37 +1256,6 @@ jobs: runs-on: ubuntu-22.04 steps: - - name: Fetch GITHUB_RUN_ID and COMMIT_SHA for the last merged release PR - id: fetch-last-release-pr-info - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - branch_name_and_pr_number=$(gh pr list \ - --repo "${GITHUB_REPOSITORY}" \ - --base release \ - --state merged \ - --limit 10 \ - --json mergeCommit,headRefName,number \ - --jq ".[] | select(.mergeCommit.oid==\"${GITHUB_SHA}\") | { branch_name: .headRefName, pr_number: .number }") - branch_name=$(echo "${branch_name_and_pr_number}" | jq -r '.branch_name') - pr_number=$(echo "${branch_name_and_pr_number}" | jq -r '.pr_number') - - run_id=$(gh run list \ - --repo "${GITHUB_REPOSITORY}" \ - --workflow build_and_test.yml \ - --branch "${branch_name}" \ - --json databaseId \ - --limit 1 \ - --jq '.[].databaseId') - - last_commit_sha=$(gh pr view "${pr_number}" \ - --repo "${GITHUB_REPOSITORY}" \ - --json commits \ - --jq '.commits[-1].oid') - - echo "run-id=${run_id}" | tee -a ${GITHUB_OUTPUT} - echo "commit-sha=${last_commit_sha}" | tee -a ${GITHUB_OUTPUT} - - uses: aws-actions/configure-aws-credentials@v4 with: aws-region: eu-central-1 @@ -1286,8 +1266,8 @@ jobs: env: BUCKET: neon-github-public-dev AWS_REGION: eu-central-1 - COMMIT_SHA: ${{ steps.fetch-last-release-pr-info.outputs.commit-sha }} - RUN_ID: ${{ steps.fetch-last-release-pr-info.outputs.run-id }} + COMMIT_SHA: ${{ github.sha }} + RUN_ID: ${{ needs.meta.outputs.release-pr-run-id }} run: | old_prefix="artifacts/${COMMIT_SHA}/${RUN_ID}" new_prefix="artifacts/latest" @@ -1376,5 +1356,5 @@ jobs: || needs.files-changed.result == 'skipped' || (needs.push-compute-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)) || (needs.push-neon-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)) - || needs.test-images.result == 'skipped' + || (needs.test-images.result == 'skipped' && contains(fromJSON('["push-main", "pr", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind)) || (needs.trigger-custom-extensions-build-and-wait.result == 'skipped' && contains(fromJSON('["push-main", "pr", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)) diff --git a/.github/workflows/fast-forward.yml b/.github/workflows/fast-forward.yml new file mode 100644 index 0000000000..bc63ff120d --- /dev/null +++ b/.github/workflows/fast-forward.yml @@ -0,0 +1,36 @@ +name: Fast forward merge +on: + pull_request: + types: [labeled] + branches: + - release + - release-proxy + - release-compute + +jobs: + fast-forward: + if: ${{ github.event.label.name == 'fast-forward' }} + runs-on: ubuntu-22.04 + + steps: + - name: Remove fast-forward label to PR + env: + GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} + run: | + gh pr edit ${{ github.event.pull_request.number }} --repo "${GITHUB_REPOSITORY}" --remove-label "fast-forward" + + - name: Fast forwarding + uses: sequoia-pgp/fast-forward@ea7628bedcb0b0b96e94383ada458d812fca4979 + # See https://docs.github.com/en/graphql/reference/enums#mergestatestatus + if: ${{ github.event.pull_request.mergeable_state == 'clean' }} + with: + merge: true + comment: on-error + github_token: ${{ secrets.CI_ACCESS_TOKEN }} + + - name: Comment if mergeable_state is not clean + if: ${{ github.event.pull_request.mergeable_state != 'clean' }} + run: | + gh pr comment ${{ github.event.pull_request.number }} \ + --repo "${GITHUB_REPOSITORY}" \ + --body "Not trying to forward pull-request, because \`mergeable_state\` is \`${{ github.event.pull_request.mergeable_state }}\`, not \`clean\`." diff --git a/.github/workflows/lint-release-pr.yml b/.github/workflows/lint-release-pr.yml new file mode 100644 index 0000000000..f12ddfe377 --- /dev/null +++ b/.github/workflows/lint-release-pr.yml @@ -0,0 +1,23 @@ +name: Lint Release PR + +on: + pull_request: + branches: + - release + - release-proxy + - release-compute + +jobs: + lint-release-pr: + runs-on: ubuntu-22.04 + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch full history for git operations + + - name: Run lint script + env: + RELEASE_BRANCH: ${{ github.base_ref }} + run: | + ./.github/scripts/lint-release-pr.sh diff --git a/.github/workflows/pre-merge-checks.yml b/.github/workflows/pre-merge-checks.yml index c47b3fe0de..1e81550314 100644 --- a/.github/workflows/pre-merge-checks.yml +++ b/.github/workflows/pre-merge-checks.yml @@ -8,8 +8,6 @@ on: - .github/workflows/build-build-tools-image.yml - .github/workflows/pre-merge-checks.yml merge_group: - branches: - - main defaults: run: @@ -19,11 +17,13 @@ defaults: permissions: {} jobs: - get-changed-files: + meta: runs-on: ubuntu-22.04 outputs: python-changed: ${{ steps.python-src.outputs.any_changed }} rust-changed: ${{ steps.rust-src.outputs.any_changed }} + branch: ${{ steps.group-metadata.outputs.branch }} + pr-number: ${{ steps.group-metadata.outputs.pr-number }} steps: - uses: actions/checkout@v4 @@ -58,12 +58,20 @@ jobs: echo "${PYTHON_CHANGED_FILES}" echo "${RUST_CHANGED_FILES}" + - name: Merge group metadata + if: ${{ github.event_name == 'merge_group' }} + id: group-metadata + env: + MERGE_QUEUE_REF: ${{ github.event.merge_group.head_ref }} + run: | + echo $MERGE_QUEUE_REF | jq -Rr 'capture("refs/heads/gh-readonly-queue/(?.*)/pr-(?[0-9]+)-[0-9a-f]{40}") | ["branch=" + .branch, "pr-number=" + .pr_number] | .[]' | tee -a "${GITHUB_OUTPUT}" + build-build-tools-image: if: | false - || needs.get-changed-files.outputs.python-changed == 'true' - || needs.get-changed-files.outputs.rust-changed == 'true' - needs: [ get-changed-files ] + || needs.meta.outputs.python-changed == 'true' + || needs.meta.outputs.rust-changed == 'true' + needs: [ meta ] uses: ./.github/workflows/build-build-tools-image.yml with: # Build only one combination to save time @@ -72,8 +80,8 @@ jobs: secrets: inherit check-codestyle-python: - if: needs.get-changed-files.outputs.python-changed == 'true' - needs: [ get-changed-files, build-build-tools-image ] + if: needs.meta.outputs.python-changed == 'true' + needs: [ meta, build-build-tools-image ] uses: ./.github/workflows/_check-codestyle-python.yml with: # `-bookworm-x64` suffix should match the combination in `build-build-tools-image` @@ -81,8 +89,8 @@ jobs: secrets: inherit check-codestyle-rust: - if: needs.get-changed-files.outputs.rust-changed == 'true' - needs: [ get-changed-files, build-build-tools-image ] + if: needs.meta.outputs.rust-changed == 'true' + needs: [ meta, build-build-tools-image ] uses: ./.github/workflows/_check-codestyle-rust.yml with: # `-bookworm-x64` suffix should match the combination in `build-build-tools-image` @@ -101,7 +109,7 @@ jobs: statuses: write # for `github.repos.createCommitStatus(...)` contents: write needs: - - get-changed-files + - meta - check-codestyle-python - check-codestyle-rust runs-on: ubuntu-22.04 @@ -129,7 +137,20 @@ jobs: run: exit 1 if: | false - || (needs.check-codestyle-python.result == 'skipped' && needs.get-changed-files.outputs.python-changed == 'true') - || (needs.check-codestyle-rust.result == 'skipped' && needs.get-changed-files.outputs.rust-changed == 'true') + || (github.event_name == 'merge_group' && needs.meta.outputs.branch != 'main') + || (needs.check-codestyle-python.result == 'skipped' && needs.meta.outputs.python-changed == 'true') + || (needs.check-codestyle-rust.result == 'skipped' && needs.meta.outputs.rust-changed == 'true') || contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + + - name: Add fast-forward label to PR to trigger fast-forward merge + if: >- + ${{ + always() + && github.event_name == 'merge_group' + && contains(fromJson('["release", "release-proxy", "release-compute"]'), github.base_ref) + }} + env: + GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} + run: >- + gh pr edit ${{ needs.meta.outputs.pr-number }} --repo "${GITHUB_REPOSITORY}" --add-label "fast-forward" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 919846ce44..a88ddecd0a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: uses: ./.github/workflows/_create-release-pr.yml with: component-name: 'Storage' - release-branch: 'release' + source-branch: ${{ github.ref_name }} secrets: ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }} @@ -51,7 +51,7 @@ jobs: uses: ./.github/workflows/_create-release-pr.yml with: component-name: 'Proxy' - release-branch: 'release-proxy' + source-branch: ${{ github.ref_name }} secrets: ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }} @@ -64,6 +64,6 @@ jobs: uses: ./.github/workflows/_create-release-pr.yml with: component-name: 'Compute' - release-branch: 'release-compute' + source-branch: ${{ github.ref_name }} secrets: ci-access-token: ${{ secrets.CI_ACCESS_TOKEN }}