rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-06 13:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(ci): don't build storage on compute-releases and vice versa (#10841)

Browse Source 
## Problem
Release CI is slow, because we're doing unnecessary work, for example
building compute images on storage releases and vice versa.

## Summary of changes
- Extract tag generation into reusable workflow and extend it with
fetching of previous component releases
- Don't build neon images on compute releases and don't build compute
images on proxy and storage releases
- Reuse images from previous releases for tests on branches where we
don't build those images

## Open questions
- We differentiate between `TAG` and `COMPUTE_TAG` in a few places, but
we don't differentiate between storage and proxy releases. Since they
use the same image, this will continue to work, but I'm not sure this is
what we want.
This commit is contained in:
JC Grünhage
2025-02-26 18:17:26 +01:00
committed by GitHub
parent a138a6de9b
commit 8dfa8f0b94
9 changed files with 317 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
scripts/generate_image_maps.py → .github/scripts/generate_image_maps.py vendored
View File

25
.github/scripts/previous-releases.jq vendored Normal file
View File

@@ -0,0 +1,25 @@
# Expects response from https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#list-releases as input,
# with tag names `release` for storage, `release-compute` for compute and `release-proxy` for proxy releases.
# Extract only the `tag_name` field from each release object
[ .[].tag_name ]
# Transform each tag name into a structured object using regex capture
| reduce map(
capture("^(?<full>release(-(?<component>proxy|compute))?-(?<version>\\d+))$")
| {
component: (.component // "storage"), # Default to "storage" if no component is specified
version: (.version | tonumber), # Convert the version number to an integer
full: .full # Store the full tag name for final output
}
)[] as $entry # Loop over the transformed list
# Accumulate the latest (highest-numbered) version for each component
({};
.[$entry.component] |= (if . == null or $entry.version > .version then $entry else . end))
# Convert the resulting object into an array of formatted strings
| to_entries
| map("\(.key)=\(.value.full)")
# Output each string separately
| .[]

0
scripts/push_with_image_map.py → .github/scripts/push_with_image_map.py vendored
View File

103
.github/workflows/_meta.yml vendored Normal file
View File

@@ -0,0 +1,103 @@
name: Generate run metadata
on:
workflow_call:
inputs:
github-event-name:
type: string
required: true
outputs:
build-tag:
description: "Tag for the current workflow run"
value: ${{ jobs.tags.outputs.build-tag }}
previous-storage-release:
description: "Tag of the last storage release"
value: ${{ jobs.tags.outputs.storage }}
previous-proxy-release:
description: "Tag of the last proxy release"
value: ${{ jobs.tags.outputs.proxy }}
previous-compute-release:
description: "Tag of the last compute release"
value: ${{ jobs.tags.outputs.compute }}
run-kind:
description: "The kind of run we're currently in. Will be one of `pr-main`, `push-main`, `storage-rc`, `storage-release`, `proxy-rc`, `proxy-release`, `compute-rc`, `compute-release` or `merge_queue`"
value: ${{ jobs.tags.outputs.run-kind }}
permissions: {}
jobs:
tags:
runs-on: ubuntu-22.04
outputs:
build-tag: ${{ steps.build-tag.outputs.tag }}
compute: ${{ steps.previous-releases.outputs.compute }}
proxy: ${{ steps.previous-releases.outputs.proxy }}
storage: ${{ steps.previous-releases.outputs.storage }}
run-kind: ${{ steps.run-kind.outputs.run-kind }}
permissions:
contents: read
steps:
# Need `fetch-depth: 0` to count the number of commits in the branch
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get run kind
id: run-kind
env:
RUN_KIND: >-
${{
false
|| (inputs.github-event-name == 'push' && github.ref_name == 'main') && 'push-main'
|| (inputs.github-event-name == 'push' && github.ref_name == 'release') && 'storage-release'
|| (inputs.github-event-name == 'push' && github.ref_name == 'release-compute') && 'compute-release'
|| (inputs.github-event-name == 'push' && github.ref_name == 'release-proxy') && 'proxy-release'
|| (inputs.github-event-name == 'pull_request' && github.base_ref == 'main') && 'pr-main'
|| (inputs.github-event-name == 'pull_request' && github.base_ref == 'release') && 'storage-rc-pr'
|| (inputs.github-event-name == 'pull_request' && github.base_ref == 'release-compute') && 'compute-rc-pr'
|| (inputs.github-event-name == 'pull_request' && github.base_ref == 'release-proxy') && 'proxy-rc-pr'
|| 'unknown'
}}
run: |
echo "run-kind=$RUN_KIND" | tee -a $GITHUB_OUTPUT
- name: Get build tag
id: build-tag
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
CURRENT_BRANCH: ${{ github.head_ref || github.ref_name }}
CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
RUN_KIND: ${{ steps.run-kind.outputs.run-kind }}
run: |
case $RUN_KIND in
push-main)
echo "tag=$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
;;
storage-release)
echo "tag=release-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
;;
proxy-release)
echo "tag=release-proxy-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
;;
compute-release)
echo "tag=release-compute-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
;;
pr-main|storage-rc-pr|compute-rc-pr|proxy-rc-pr)
BUILD_AND_TEST_RUN_ID=$(gh run list -b $CURRENT_BRANCH -c $CURRENT_SHA -w 'Build and Test' -L 1 --json databaseId --jq '.[].databaseId')
echo "tag=$BUILD_AND_TEST_RUN_ID" | tee -a $GITHUB_OUTPUT
;;
*)
echo "Unexpected RUN_KIND ('${RUN_KIND}'), failing to assign build-tag!"
exit 1
esac
- name: Get the previous release-tags
id: previous-releases
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh api --paginate \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"/repos/${GITHUB_REPOSITORY}/releases" \
| jq -f .github/scripts/previous-releases.jq -r \
| tee -a "${GITHUB_OUTPUT}"

4
.github/workflows/_push-to-container-registry.yml vendored
View File

@@ -51,7 +51,7 @@ jobs:
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
sparse-checkout: scripts/push_with_image_map.py sparse-checkout: .github/scripts/push_with_image_map.py
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false
- name: Print image-map - name: Print image-map
@@ -99,6 +99,6 @@ jobs:
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
- name: Copy docker images to target registries - name: Copy docker images to target registries
run: python scripts/push_with_image_map.py run: python3 .github/scripts/push_with_image_map.py
env: env:
IMAGE_MAP: ${{ inputs.image-map }} IMAGE_MAP: ${{ inputs.image-map }}

306
.github/workflows/build_and_test.yml vendored
View File

@@ -65,38 +65,11 @@ jobs:
token: ${{ secrets.GITHUB_TOKEN }} token: ${{ secrets.GITHUB_TOKEN }}
filters: .github/file-filters.yaml filters: .github/file-filters.yaml
tag: meta:
needs: [ check-permissions ] needs: [ check-permissions ]
runs-on: [ self-hosted, small ] uses: ./.github/workflows/_meta.yml
container: ${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/base:pinned with:
outputs: github-event-name: ${{ github.event_name }}
build-tag: ${{steps.build-tag.outputs.tag}}
steps:
# Need `fetch-depth: 0` to count the number of commits in the branch
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get build tag
run: |
echo run:$GITHUB_RUN_ID
echo ref:$GITHUB_REF_NAME
echo rev:$(git rev-list --count HEAD)
if [[ "$GITHUB_REF_NAME" == "main" ]]; then
echo "tag=$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
echo "tag=release-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then
echo "tag=release-proxy-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
elif [[ "$GITHUB_REF_NAME" == "release-compute" ]]; then
echo "tag=release-compute-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
else
echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release', 'release-proxy', 'release-compute'"
echo "tag=$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
fi
shell: bash
id: build-tag
build-build-tools-image: build-build-tools-image:
needs: [ check-permissions ] needs: [ check-permissions ]
@@ -199,7 +172,7 @@ jobs:
secrets: inherit secrets: inherit
build-and-test-locally: build-and-test-locally:
needs: [ tag, build-build-tools-image ] needs: [ meta, build-build-tools-image ]
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@@ -213,7 +186,7 @@ jobs:
with: with:
arch: ${{ matrix.arch }} arch: ${{ matrix.arch }}
build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
build-tag: ${{ needs.tag.outputs.build-tag }} build-tag: ${{ needs.meta.outputs.build-tag }}
build-type: ${{ matrix.build-type }} build-type: ${{ matrix.build-type }}
# Run tests on all Postgres versions in release builds and only on the latest version in debug builds. # Run tests on all Postgres versions in release builds and only on the latest version in debug builds.
# Run without LFC on v17 release and debug builds only. For all the other cases LFC is enabled. # Run without LFC on v17 release and debug builds only. For all the other cases LFC is enabled.
@@ -497,13 +470,24 @@ jobs:
}) })
trigger-e2e-tests: trigger-e2e-tests:
if: ${{ !github.event.pull_request.draft || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') || github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' }} # Depends on jobs that can get skipped
needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, tag ] if: >-
${{
(
!github.event.pull_request.draft
|| contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft')
|| contains(fromJSON('["push-main", "storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind)
) && !failure() && !cancelled()
}}
needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, meta ]
uses: ./.github/workflows/trigger-e2e-tests.yml uses: ./.github/workflows/trigger-e2e-tests.yml
with:
github-event-name: ${{ github.event_name }}
secrets: inherit secrets: inherit
neon-image-arch: neon-image-arch:
needs: [ check-permissions, build-build-tools-image, tag ] needs: [ check-permissions, build-build-tools-image, meta ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
strategy: strategy:
matrix: matrix:
arch: [ x64, arm64 ] arch: [ x64, arm64 ]
@@ -539,7 +523,7 @@ jobs:
build-args: | build-args: |
ADDITIONAL_RUSTFLAGS=${{ matrix.arch == 'arm64' && '-Ctarget-feature=+lse -Ctarget-cpu=neoverse-n1' || '' }} ADDITIONAL_RUSTFLAGS=${{ matrix.arch == 'arm64' && '-Ctarget-feature=+lse -Ctarget-cpu=neoverse-n1' || '' }}
GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }}
BUILD_TAG=${{ needs.tag.outputs.build-tag }} BUILD_TAG=${{ needs.meta.outputs.build-tag }}
TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-bookworm TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-bookworm
DEBIAN_VERSION=bookworm DEBIAN_VERSION=bookworm
provenance: false provenance: false
@@ -549,10 +533,11 @@ jobs:
cache-from: type=registry,ref=cache.neon.build/neon:cache-bookworm-${{ matrix.arch }} cache-from: type=registry,ref=cache.neon.build/neon:cache-bookworm-${{ matrix.arch }}
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/neon:cache-{0}-{1},mode=max', 'bookworm', matrix.arch) || '' }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/neon:cache-{0}-{1},mode=max', 'bookworm', matrix.arch) || '' }}
tags: | tags: |
neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-${{ matrix.arch }} neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-${{ matrix.arch }}
neon-image: neon-image:
needs: [ neon-image-arch, tag ] needs: [ neon-image-arch, meta ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
permissions: permissions:
id-token: write # aws-actions/configure-aws-credentials id-token: write # aws-actions/configure-aws-credentials
@@ -567,13 +552,14 @@ jobs:
- name: Create multi-arch image - name: Create multi-arch image
run: | run: |
docker buildx imagetools create -t neondatabase/neon:${{ needs.tag.outputs.build-tag }} \ docker buildx imagetools create -t neondatabase/neon:${{ needs.meta.outputs.build-tag }} \
-t neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm \ -t neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm \
neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-x64 \ neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-x64 \
neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-arm64 neondatabase/neon:${{ needs.meta.outputs.build-tag }}-bookworm-arm64
compute-node-image-arch: compute-node-image-arch:
needs: [ check-permissions, build-build-tools-image, tag ] needs: [ check-permissions, build-build-tools-image, meta ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
permissions: permissions:
id-token: write # aws-actions/configure-aws-credentials id-token: write # aws-actions/configure-aws-credentials
statuses: write statuses: write
@@ -631,7 +617,7 @@ jobs:
build-args: | build-args: |
GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }}
PG_VERSION=${{ matrix.version.pg }} PG_VERSION=${{ matrix.version.pg }}
BUILD_TAG=${{ needs.tag.outputs.build-tag }} BUILD_TAG=${{ needs.meta.outputs.build-tag }}
TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-${{ matrix.version.debian }} TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-${{ matrix.version.debian }}
DEBIAN_VERSION=${{ matrix.version.debian }} DEBIAN_VERSION=${{ matrix.version.debian }}
provenance: false provenance: false
@@ -641,7 +627,7 @@ jobs:
cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }}
cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-node-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }} cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-node-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }}
tags: | tags: |
neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-${{ matrix.arch }} neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-${{ matrix.arch }}
- name: Build neon extensions test image - name: Build neon extensions test image
if: matrix.version.pg >= 'v16' if: matrix.version.pg >= 'v16'
@@ -651,7 +637,7 @@ jobs:
build-args: | build-args: |
GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }}
PG_VERSION=${{ matrix.version.pg }} PG_VERSION=${{ matrix.version.pg }}
BUILD_TAG=${{ needs.tag.outputs.build-tag }} BUILD_TAG=${{ needs.meta.outputs.build-tag }}
TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-${{ matrix.version.debian }} TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-${{ matrix.version.debian }}
DEBIAN_VERSION=${{ matrix.version.debian }} DEBIAN_VERSION=${{ matrix.version.debian }}
provenance: false provenance: false
@@ -661,10 +647,11 @@ jobs:
target: extension-tests target: extension-tests
cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }}
tags: | tags: |
neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{needs.tag.outputs.build-tag}}-${{ matrix.version.debian }}-${{ matrix.arch }} neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{needs.meta.outputs.build-tag}}-${{ matrix.version.debian }}-${{ matrix.arch }}
compute-node-image: compute-node-image:
needs: [ compute-node-image-arch, tag ] needs: [ compute-node-image-arch, meta ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
permissions: permissions:
id-token: write # aws-actions/configure-aws-credentials id-token: write # aws-actions/configure-aws-credentials
statuses: write statuses: write
@@ -692,21 +679,22 @@ jobs:
- name: Create multi-arch compute-node image - name: Create multi-arch compute-node image
run: | run: |
docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ docker buildx imagetools create -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \ -t neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \
neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
- name: Create multi-arch neon-test-extensions image - name: Create multi-arch neon-test-extensions image
if: matrix.version.pg >= 'v16' if: matrix.version.pg >= 'v16'
run: | run: |
docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ docker buildx imagetools create -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \ -t neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }} \
neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}-${{ matrix.version.debian }}-arm64
vm-compute-node-image: vm-compute-node-image:
needs: [ check-permissions, tag, compute-node-image ] needs: [ check-permissions, meta, compute-node-image ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
runs-on: [ self-hosted, large ] runs-on: [ self-hosted, large ]
strategy: strategy:
fail-fast: false fail-fast: false
@@ -742,23 +730,25 @@ jobs:
# it won't have the proper authentication (written at v0.6.0) # it won't have the proper authentication (written at v0.6.0)
- name: Pulling compute-node image - name: Pulling compute-node image
run: | run: |
docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} docker pull neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}
- name: Build vm image - name: Build vm image
run: | run: |
./vm-builder \ ./vm-builder \
-size=2G \ -size=2G \
-spec=compute/vm-image-spec-${{ matrix.version.debian }}.yaml \ -spec=compute/vm-image-spec-${{ matrix.version.debian }}.yaml \
-src=neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ -src=neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ -dst=neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }} \
-target-arch=linux/amd64 -target-arch=linux/amd64
- name: Pushing vm-compute-node image - name: Pushing vm-compute-node image
run: | run: |
docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} docker push neondatabase/vm-compute-node-${{ matrix.version.pg }}:${{ needs.meta.outputs.build-tag }}
test-images: test-images:
needs: [ check-permissions, tag, neon-image, compute-node-image ] needs: [ check-permissions, meta, neon-image, compute-node-image ]
# Depends on jobs that can get skipped
if: "!failure() && !cancelled()"
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
@@ -776,17 +766,6 @@ jobs:
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
- name: Get the last compute release tag
id: get-last-compute-release-tag
env:
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
run: |
tag=$(gh api -q '[.[].tag_name | select(startswith("release-compute"))][0]'\
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"/repos/${{ github.repository }}/releases")
echo tag=${tag} >> ${GITHUB_OUTPUT}
# `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library. # `neondatabase/neon` contains multiple binaries, all of them use the same input for the version into the same version formatting library.
# Pick pageserver as currently the only binary with extra "version" features printed in the string to verify. # Pick pageserver as currently the only binary with extra "version" features printed in the string to verify.
# Regular pageserver version string looks like # Regular pageserver version string looks like
@@ -796,8 +775,9 @@ jobs:
# Ensure that we don't have bad versions. # Ensure that we don't have bad versions.
- name: Verify image versions - name: Verify image versions
shell: bash # ensure no set -e for better error messages shell: bash # ensure no set -e for better error messages
if: ${{ contains(fromJSON('["push-main", "pr-main", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
run: | run: |
pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.tag.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version") pageserver_version=$(docker run --rm neondatabase/neon:${{ needs.meta.outputs.build-tag }} "/bin/sh" "-c" "/usr/local/bin/pageserver --version")
echo "Pageserver version string: $pageserver_version" echo "Pageserver version string: $pageserver_version"
@@ -814,7 +794,24 @@ jobs:
- name: Verify docker-compose example and test extensions - name: Verify docker-compose example and test extensions
timeout-minutes: 20 timeout-minutes: 20
env: env:
TAG: ${{needs.tag.outputs.build-tag}} TAG: >-
${{
contains(fromJSON('["compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)
&& needs.meta.outputs.previous-storage-release
|| needs.meta.outputs.build-tag
}}
COMPUTE_TAG: >-
${{
contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)
&& needs.meta.outputs.previous-compute-release
|| needs.meta.outputs.build-tag
}}
TEST_EXTENSIONS_TAG: >-
${{
contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)
&& 'latest'
|| needs.meta.outputs.build-tag
}}
TEST_VERSION_ONLY: ${{ matrix.pg_version }} TEST_VERSION_ONLY: ${{ matrix.pg_version }}
run: ./docker-compose/docker_compose_test.sh run: ./docker-compose/docker_compose_test.sh
@@ -826,10 +823,17 @@ jobs:
- name: Test extension upgrade - name: Test extension upgrade
timeout-minutes: 20 timeout-minutes: 20
if: ${{ needs.tag.outputs.build-tag == github.run_id }} if: ${{ contains(fromJSON('["pr-main", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
env: env:
NEWTAG: ${{ needs.tag.outputs.build-tag }} TAG: >-
OLDTAG: ${{ steps.get-last-compute-release-tag.outputs.tag }} ${{
false
|| needs.meta.outputs.run-kind == 'pr-main' && needs.meta.outputs.build-tag
|| needs.meta.outputs.run-kind == 'compute-rc-pr' && needs.meta.outputs.previous-storage-release
}}
TEST_EXTENSIONS_TAG: latest
NEW_COMPUTE_TAG: ${{ needs.meta.outputs.build-tag }}
OLD_COMPUTE_TAG: ${{ needs.meta.outputs.previous-compute-release }}
run: ./docker-compose/test_extensions_upgrade.sh run: ./docker-compose/test_extensions_upgrade.sh
- name: Print logs and clean up - name: Print logs and clean up
@@ -839,7 +843,7 @@ jobs:
docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml down docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml down
generate-image-maps: generate-image-maps:
needs: [ tag ] needs: [ meta ]
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
outputs: outputs:
neon-dev: ${{ steps.generate.outputs.neon-dev }} neon-dev: ${{ steps.generate.outputs.neon-dev }}
@@ -849,14 +853,14 @@ jobs:
steps: steps:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
with: with:
sparse-checkout: scripts/generate_image_maps.py sparse-checkout: .github/scripts/generate_image_maps.py
sparse-checkout-cone-mode: false sparse-checkout-cone-mode: false
- name: Generate Image Maps - name: Generate Image Maps
id: generate id: generate
run: python scripts/generate_image_maps.py run: python3 .github/scripts/generate_image_maps.py
env: env:
BUILD_TAG: "${{ needs.tag.outputs.build-tag }}" BUILD_TAG: "${{ needs.meta.outputs.build-tag }}"
BRANCH: "${{ github.ref_name }}" BRANCH: "${{ github.ref_name }}"
DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}" DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}"
PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}" PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}"
@@ -865,7 +869,8 @@ jobs:
AWS_REGION: "${{ vars.AWS_ECR_REGION }}" AWS_REGION: "${{ vars.AWS_ECR_REGION }}"
push-neon-image-dev: push-neon-image-dev:
needs: [ generate-image-maps, neon-image ] needs: [ meta, generate-image-maps, neon-image ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind) }}
uses: ./.github/workflows/_push-to-container-registry.yml uses: ./.github/workflows/_push-to-container-registry.yml
permissions: permissions:
id-token: write # Required for aws/azure login id-token: write # Required for aws/azure login
@@ -882,7 +887,8 @@ jobs:
secrets: inherit secrets: inherit
push-compute-image-dev: push-compute-image-dev:
needs: [ generate-image-maps, vm-compute-node-image ] needs: [ meta, generate-image-maps, vm-compute-node-image ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
uses: ./.github/workflows/_push-to-container-registry.yml uses: ./.github/workflows/_push-to-container-registry.yml
permissions: permissions:
id-token: write # Required for aws/azure login id-token: write # Required for aws/azure login
@@ -899,8 +905,8 @@ jobs:
secrets: inherit secrets: inherit
push-neon-image-prod: push-neon-image-prod:
if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' needs: [ meta, generate-image-maps, neon-image, test-images ]
needs: [ generate-image-maps, neon-image, test-images ] if: ${{ contains(fromJSON('["storage-release", "proxy-release"]'), needs.meta.outputs.run-kind) }}
uses: ./.github/workflows/_push-to-container-registry.yml uses: ./.github/workflows/_push-to-container-registry.yml
permissions: permissions:
id-token: write # Required for aws/azure login id-token: write # Required for aws/azure login
@@ -917,8 +923,8 @@ jobs:
secrets: inherit secrets: inherit
push-compute-image-prod: push-compute-image-prod:
if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' needs: [ meta, generate-image-maps, vm-compute-node-image, test-images ]
needs: [ generate-image-maps, vm-compute-node-image, test-images ] if: ${{ needs.meta.outputs.run-kind == 'compute-release' }}
uses: ./.github/workflows/_push-to-container-registry.yml uses: ./.github/workflows/_push-to-container-registry.yml
permissions: permissions:
id-token: write # Required for aws/azure login id-token: write # Required for aws/azure login
@@ -937,18 +943,19 @@ jobs:
# This is a bit of a special case so we're not using a generated image map. # This is a bit of a special case so we're not using a generated image map.
add-latest-tag-to-neon-extensions-test-image: add-latest-tag-to-neon-extensions-test-image:
if: github.ref_name == 'main' if: github.ref_name == 'main'
needs: [ tag, compute-node-image ] needs: [ meta, compute-node-image ]
uses: ./.github/workflows/_push-to-container-registry.yml uses: ./.github/workflows/_push-to-container-registry.yml
with: with:
image-map: | image-map: |
{ {
"docker.io/neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v16:latest"], "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.meta.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v16:latest"],
"docker.io/neondatabase/neon-test-extensions-v17:${{ needs.tag.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v17:latest"] "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.meta.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v17:latest"]
} }
secrets: inherit secrets: inherit
trigger-custom-extensions-build-and-wait: trigger-custom-extensions-build-and-wait:
needs: [ check-permissions, tag ] needs: [ check-permissions, meta ]
if: ${{ contains(fromJSON('["push-main", "pr-main", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
permissions: permissions:
id-token: write # aws-actions/configure-aws-credentials id-token: write # aws-actions/configure-aws-credentials
@@ -983,7 +990,7 @@ jobs:
\"ci_job_name\": \"build-and-upload-extensions\", \"ci_job_name\": \"build-and-upload-extensions\",
\"commit_hash\": \"$COMMIT_SHA\", \"commit_hash\": \"$COMMIT_SHA\",
\"remote_repo\": \"${{ github.repository }}\", \"remote_repo\": \"${{ github.repository }}\",
\"compute_image_tag\": \"${{ needs.tag.outputs.build-tag }}\", \"compute_image_tag\": \"${{ needs.meta.outputs.build-tag }}\",
\"remote_branch_name\": \"${{ github.ref_name }}\" \"remote_branch_name\": \"${{ github.ref_name }}\"
} }
}" }"
@@ -1027,9 +1034,9 @@ jobs:
exit 1 exit 1
deploy: deploy:
needs: [ check-permissions, push-neon-image-prod, push-compute-image-prod, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait ] needs: [ check-permissions, push-neon-image-prod, push-compute-image-prod, meta, build-and-test-locally, trigger-custom-extensions-build-and-wait ]
# `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod` # `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-neon-image-prod` and `push-compute-image-prod`
if: (github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute') && !failure() && !cancelled() if: ${{ contains(fromJSON('["push-main", "storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind) && !failure() && !cancelled() }}
permissions: permissions:
id-token: write # aws-actions/configure-aws-credentials id-token: write # aws-actions/configure-aws-credentials
statuses: write statuses: write
@@ -1040,108 +1047,103 @@ jobs:
- uses: actions/checkout@v4 - uses: actions/checkout@v4
- name: Create git tag and GitHub release - name: Create git tag and GitHub release
if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' if: ${{ contains(fromJSON('["storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind) }}
uses: actions/github-script@v7 uses: actions/github-script@v7
env:
TAG: "${{ needs.meta.outputs.build-tag }}"
BRANCH: "${{ github.ref_name }}"
PREVIOUS_RELEASE: >-
${{
false
|| needs.meta.outputs.run-kind == 'storage-release' && needs.meta.outputs.previous-storage-release
|| needs.meta.outputs.run-kind == 'proxy-release' && needs.meta.outputs.previous-proxy-release
|| needs.meta.outputs.run-kind == 'compute-release' && needs.meta.outputs.previous-compute-release
|| 'unknown'
}}
with: with:
retries: 5 retries: 5
script: | script: |
const tag = "${{ needs.tag.outputs.build-tag }}"; const { TAG, BRANCH, PREVIOUS_RELEASE } = process.env
const branch = "${{ github.ref_name }}";
try { try {
const existingRef = await github.rest.git.getRef({ const existingRef = await github.rest.git.getRef({
owner: context.repo.owner, owner: context.repo.owner,
repo: context.repo.repo, repo: context.repo.repo,
ref: `tags/${tag}`, ref: `tags/${TAG}`,
}); });
if (existingRef.data.object.sha !== context.sha) { if (existingRef.data.object.sha !== context.sha) {
throw new Error(`Tag ${tag} already exists but points to a different commit (expected: ${context.sha}, actual: ${existingRef.data.object.sha}).`); throw new Error(`Tag ${TAG} already exists but points to a different commit (expected: ${context.sha}, actual: ${existingRef.data.object.sha}).`);
} }
console.log(`Tag ${tag} already exists and points to ${context.sha} as expected.`); console.log(`Tag ${TAG} already exists and points to ${context.sha} as expected.`);
} catch (error) { } catch (error) {
if (error.status !== 404) { if (error.status !== 404) {
throw error; throw error;
} }
console.log(`Tag ${tag} does not exist. Creating it...`); console.log(`Tag ${TAG} does not exist. Creating it...`);
await github.rest.git.createRef({ await github.rest.git.createRef({
owner: context.repo.owner, owner: context.repo.owner,
repo: context.repo.repo, repo: context.repo.repo,
ref: `refs/tags/${tag}`, ref: `refs/tags/${TAG}`,
sha: context.sha, sha: context.sha,
}); });
console.log(`Tag ${tag} created successfully.`); console.log(`Tag ${TAG} created successfully.`);
} }
try { try {
const existingRelease = await github.rest.repos.getReleaseByTag({ const existingRelease = await github.rest.repos.getReleaseByTag({
owner: context.repo.owner, owner: context.repo.owner,
repo: context.repo.repo, repo: context.repo.repo,
tag: tag, tag: TAG,
}); });
console.log(`Release for tag ${tag} already exists (ID: ${existingRelease.data.id}).`); console.log(`Release for tag ${TAG} already exists (ID: ${existingRelease.data.id}).`);
} catch (error) { } catch (error) {
if (error.status !== 404) { if (error.status !== 404) {
throw error; throw error;
} }
console.log(`Release for tag ${tag} does not exist. Creating it...`); console.log(`Release for tag ${TAG} does not exist. Creating it...`);
// Find the PR number using the commit SHA // Find the PR number using the commit SHA
const pullRequests = await github.rest.pulls.list({ const pullRequests = await github.rest.pulls.list({
owner: context.repo.owner, owner: context.repo.owner,
repo: context.repo.repo, repo: context.repo.repo,
state: 'closed', state: 'closed',
base: branch, base: BRANCH,
}); });
const pr = pullRequests.data.find(pr => pr.merge_commit_sha === context.sha); const pr = pullRequests.data.find(pr => pr.merge_commit_sha === context.sha);
const prNumber = pr ? pr.number : null; const prNumber = pr ? pr.number : null;
// Find the previous release on the branch
const releases = await github.rest.repos.listReleases({
owner: context.repo.owner,
repo: context.repo.repo,
per_page: 100,
});
const branchReleases = releases.data
.filter((release) => {
const regex = new RegExp(`^${branch}-\\d+$`);
return regex.test(release.tag_name) && !release.draft && !release.prerelease;
})
.sort((a, b) => new Date(b.created_at) - new Date(a.created_at));
const previousTag = branchReleases.length > 0 ? branchReleases[0].tag_name : null;
const releaseNotes = [ const releaseNotes = [
prNumber prNumber
? `Release PR https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}.` ? `Release PR https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${prNumber}.`
: 'Release PR not found.', : 'Release PR not found.',
previousTag `Diff with the previous release https://github.com/${context.repo.owner}/${context.repo.repo}/compare/${PREVIOUS_RELEASE}...${TAG}.`
? `Diff with the previous release https://github.com/${context.repo.owner}/${context.repo.repo}/compare/${previousTag}...${tag}.`
: `No previous release found on branch ${branch}.`,
].join('\n\n'); ].join('\n\n');
await github.rest.repos.createRelease({ await github.rest.repos.createRelease({
owner: context.repo.owner, owner: context.repo.owner,
repo: context.repo.repo, repo: context.repo.repo,
tag_name: tag, tag_name: TAG,
body: releaseNotes, body: releaseNotes,
}); });
console.log(`Release for tag ${tag} created successfully.`); console.log(`Release for tag ${TAG} created successfully.`);
} }
- name: Trigger deploy workflow - name: Trigger deploy workflow
env: env:
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
RUN_KIND: ${{ needs.meta.outputs.run-kind }}
run: | run: |
if [[ "$GITHUB_REF_NAME" == "main" ]]; then case ${RUN_KIND} in
gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main -f branch=main -f dockerTag=${{needs.tag.outputs.build-tag}} -f deployPreprodRegion=false push-main)
elif [[ "$GITHUB_REF_NAME" == "release" ]]; then gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main -f branch=main -f dockerTag=${{needs.meta.outputs.build-tag}} -f deployPreprodRegion=false
;;
storage-release)
gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main \ gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main \
-f deployPgSniRouter=false \ -f deployPgSniRouter=false \
-f deployProxy=false \ -f deployProxy=false \
@@ -1149,7 +1151,7 @@ jobs:
-f deployStorageBroker=true \ -f deployStorageBroker=true \
-f deployStorageController=true \ -f deployStorageController=true \
-f branch=main \ -f branch=main \
-f dockerTag=${{needs.tag.outputs.build-tag}} \ -f dockerTag=${{needs.meta.outputs.build-tag}} \
-f deployPreprodRegion=true -f deployPreprodRegion=true
gh workflow --repo neondatabase/infra run deploy-prod.yml --ref main \ gh workflow --repo neondatabase/infra run deploy-prod.yml --ref main \
@@ -1157,8 +1159,9 @@ jobs:
-f deployStorageBroker=true \ -f deployStorageBroker=true \
-f deployStorageController=true \ -f deployStorageController=true \
-f branch=main \ -f branch=main \
-f dockerTag=${{needs.tag.outputs.build-tag}} -f dockerTag=${{needs.meta.outputs.build-tag}}
elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then ;;
proxy-release)
gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main \ gh workflow --repo neondatabase/infra run deploy-dev.yml --ref main \
-f deployPgSniRouter=true \ -f deployPgSniRouter=true \
-f deployProxy=true \ -f deployProxy=true \
@@ -1166,7 +1169,7 @@ jobs:
-f deployStorageBroker=false \ -f deployStorageBroker=false \
-f deployStorageController=false \ -f deployStorageController=false \
-f branch=main \ -f branch=main \
-f dockerTag=${{needs.tag.outputs.build-tag}} \ -f dockerTag=${{needs.meta.outputs.build-tag}} \
-f deployPreprodRegion=true -f deployPreprodRegion=true
gh workflow --repo neondatabase/infra run deploy-proxy-prod.yml --ref main \ gh workflow --repo neondatabase/infra run deploy-proxy-prod.yml --ref main \
@@ -1176,13 +1179,16 @@ jobs:
-f deployProxyScram=true \ -f deployProxyScram=true \
-f deployProxyAuthBroker=true \ -f deployProxyAuthBroker=true \
-f branch=main \ -f branch=main \
-f dockerTag=${{needs.tag.outputs.build-tag}} -f dockerTag=${{needs.meta.outputs.build-tag}}
elif [[ "$GITHUB_REF_NAME" == "release-compute" ]]; then ;;
gh workflow --repo neondatabase/infra run deploy-compute-dev.yml --ref main -f dockerTag=${{needs.tag.outputs.build-tag}} compute-release)
else gh workflow --repo neondatabase/infra run deploy-compute-dev.yml --ref main -f dockerTag=${{needs.meta.outputs.build-tag}}
echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main', 'release', 'release-proxy' or 'release-compute'" ;;
*)
echo "RUN_KIND (value '${RUN_KIND}') is not set to either 'push-main', 'storage-release', 'proxy-release' or 'compute-release'"
exit 1 exit 1
fi ;;
esac
notify-storage-release-deploy-failure: notify-storage-release-deploy-failure:
needs: [ deploy ] needs: [ deploy ]
@@ -1207,7 +1213,7 @@ jobs:
id-token: write # aws-actions/configure-aws-credentials id-token: write # aws-actions/configure-aws-credentials
statuses: write statuses: write
contents: read contents: read
# `!failure() && !cancelled()` is required because the workflow transitively depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod` # `!failure() && !cancelled()` is required because the workflow transitively depends on the job that can be skipped: `push-neon-image-prod` and `push-compute-image-prod`
if: github.ref_name == 'release' && !failure() && !cancelled() if: github.ref_name == 'release' && !failure() && !cancelled()
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
@@ -1297,7 +1303,8 @@ jobs:
pin-build-tools-image: pin-build-tools-image:
needs: [ build-build-tools-image, test-images, build-and-test-locally ] needs: [ build-build-tools-image, test-images, build-and-test-locally ]
if: github.ref_name == 'main' # `!failure() && !cancelled()` is required because the job (transitively) depends on jobs that can be skipped
if: github.ref_name == 'main' && !failure() && !cancelled()
uses: ./.github/workflows/pin-build-tools-image.yml uses: ./.github/workflows/pin-build-tools-image.yml
with: with:
from-tag: ${{ needs.build-build-tools-image.outputs.image-tag }} from-tag: ${{ needs.build-build-tools-image.outputs.image-tag }}
@@ -1316,6 +1323,7 @@ jobs:
# Format `needs` differently to make the list more readable. # Format `needs` differently to make the list more readable.
# Usually we do `needs: [...]` # Usually we do `needs: [...]`
needs: needs:
- meta
- build-and-test-locally - build-and-test-locally
- check-codestyle-python - check-codestyle-python
- check-codestyle-rust - check-codestyle-rust
@@ -1339,7 +1347,7 @@ jobs:
|| needs.check-codestyle-python.result == 'skipped' || needs.check-codestyle-python.result == 'skipped'
|| needs.check-codestyle-rust.result == 'skipped' || needs.check-codestyle-rust.result == 'skipped'
|| needs.files-changed.result == 'skipped' || needs.files-changed.result == 'skipped'
|| needs.push-compute-image-dev.result == 'skipped' || (needs.push-compute-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr-main", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind))
|| needs.push-neon-image-dev.result == 'skipped' || (needs.push-neon-image-dev.result == 'skipped' && contains(fromJSON('["push-main", "pr-main", "storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind))
|| needs.test-images.result == 'skipped' || needs.test-images.result == 'skipped'
|| needs.trigger-custom-extensions-build-and-wait.result == 'skipped' || (needs.trigger-custom-extensions-build-and-wait.result == 'skipped' && contains(fromJSON('["push-main", "pr-main", "compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind))

59
.github/workflows/trigger-e2e-tests.yml vendored
View File

@@ -5,6 +5,10 @@ on:
types: types:
- ready_for_review - ready_for_review
workflow_call: workflow_call:
inputs:
github-event-name:
type: string
required: true
defaults: defaults:
run: run:
@@ -19,7 +23,7 @@ jobs:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }} if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }}
uses: ./.github/workflows/check-permissions.yml uses: ./.github/workflows/check-permissions.yml
with: with:
github-event-name: ${{ github.event_name }} github-event-name: ${{ inputs.github-event-name || github.event_name }}
cancel-previous-e2e-tests: cancel-previous-e2e-tests:
needs: [ check-permissions ] needs: [ check-permissions ]
@@ -35,46 +39,29 @@ jobs:
run cancel-previous-in-concurrency-group.yml \ run cancel-previous-in-concurrency-group.yml \
--field concurrency_group="${{ env.E2E_CONCURRENCY_GROUP }}" --field concurrency_group="${{ env.E2E_CONCURRENCY_GROUP }}"
tag: meta:
needs: [ check-permissions ] uses: ./.github/workflows/_meta.yml
runs-on: ubuntu-22.04 with:
outputs: github-event-name: ${{ inputs.github-event-name || github.event_name }}
build-tag: ${{ steps.build-tag.outputs.tag }}
steps:
# Need `fetch-depth: 0` to count the number of commits in the branch
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Get build tag
env:
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
CURRENT_BRANCH: ${{ github.head_ref || github.ref_name }}
CURRENT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
run: |
if [[ "$GITHUB_REF_NAME" == "main" ]]; then
echo "tag=$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
echo "tag=release-$(git rev-list --count HEAD)" | tee -a $GITHUB_OUTPUT
elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then
echo "tag=release-proxy-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
elif [[ "$GITHUB_REF_NAME" == "release-compute" ]]; then
echo "tag=release-compute-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
else
echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release'"
BUILD_AND_TEST_RUN_ID=$(gh run list -b $CURRENT_BRANCH -c $CURRENT_SHA -w 'Build and Test' -L 1 --json databaseId --jq '.[].databaseId')
echo "tag=$BUILD_AND_TEST_RUN_ID" | tee -a $GITHUB_OUTPUT
fi
id: build-tag
trigger-e2e-tests: trigger-e2e-tests:
needs: [ tag ] needs: [ meta ]
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04
env: env:
EVENT_ACTION: ${{ github.event.action }} EVENT_ACTION: ${{ github.event.action }}
GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
TAG: ${{ needs.tag.outputs.build-tag }} TAG: >-
${{
contains(fromJSON('["compute-release", "compute-rc-pr"]'), needs.meta.outputs.run-kind)
&& needs.meta.outputs.previous-storage-release
|| needs.meta.outputs.build-tag
}}
COMPUTE_TAG: >-
${{
contains(fromJSON('["storage-release", "storage-rc-pr", "proxy-release", "proxy-rc-pr"]'), needs.meta.outputs.run-kind)
&& needs.meta.outputs.previous-compute-release
|| needs.meta.outputs.build-tag
}}
steps: steps:
- name: Wait for `push-{neon,compute}-image-dev` job to finish - name: Wait for `push-{neon,compute}-image-dev` job to finish
# It's important to have a timeout here, the script in the step can run infinitely # It's important to have a timeout here, the script in the step can run infinitely
@@ -157,6 +144,6 @@ jobs:
--raw-field "commit_hash=$COMMIT_SHA" \ --raw-field "commit_hash=$COMMIT_SHA" \
--raw-field "remote_repo=${GITHUB_REPOSITORY}" \ --raw-field "remote_repo=${GITHUB_REPOSITORY}" \
--raw-field "storage_image_tag=${TAG}" \ --raw-field "storage_image_tag=${TAG}" \
--raw-field "compute_image_tag=${TAG}" \ --raw-field "compute_image_tag=${COMPUTE_TAG}" \
--raw-field "concurrency_group=${E2E_CONCURRENCY_GROUP}" \ --raw-field "concurrency_group=${E2E_CONCURRENCY_GROUP}" \
--raw-field "e2e-platforms=${E2E_PLATFORMS}" --raw-field "e2e-platforms=${E2E_PLATFORMS}"

2
docker-compose/docker-compose.yml
View File

@@ -186,7 +186,7 @@ services:
neon-test-extensions: neon-test-extensions:
profiles: ["test-extensions"] profiles: ["test-extensions"]
image: ${REPOSITORY:-neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-16}:${TAG:-latest} image: ${REPOSITORY:-neondatabase}/neon-test-extensions-v${PG_TEST_VERSION:-16}:${TEST_EXTENSIONS_TAG:-${TAG:-latest}}
environment: environment:
- PGPASSWORD=cloud_admin - PGPASSWORD=cloud_admin
entrypoint: entrypoint:

12
docker-compose/test_extensions_upgrade.sh
View File

@@ -6,8 +6,8 @@ generate_id() {
local -n resvar=$1 local -n resvar=$1
printf -v resvar '%08x%08x%08x%08x' $SRANDOM $SRANDOM $SRANDOM $SRANDOM printf -v resvar '%08x%08x%08x%08x' $SRANDOM $SRANDOM $SRANDOM $SRANDOM
} }
if [ -z ${OLDTAG+x} ] || [ -z ${NEWTAG+x} ] || [ -z "${OLDTAG}" ] || [ -z "${NEWTAG}" ]; then if [ -z ${OLD_COMPUTE_TAG+x} ] || [ -z ${NEW_COMPUTE_TAG+x} ] || [ -z "${OLD_COMPUTE_TAG}" ] || [ -z "${NEW_COMPUTE_TAG}" ]; then
echo OLDTAG and NEWTAG must be defined echo OLD_COMPUTE_TAG and NEW_COMPUTE_TAG must be defined
exit 1 exit 1
fi fi
export PG_VERSION=${PG_VERSION:-16} export PG_VERSION=${PG_VERSION:-16}
@@ -47,7 +47,7 @@ EXTENSIONS='[
{"extname": "pg_repack", "extdir": "pg_repack-src"} {"extname": "pg_repack", "extdir": "pg_repack-src"}
]' ]'
EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -) EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -)
TAG=${NEWTAG} docker compose --profile test-extensions up --quiet-pull --build -d COMPUTE_TAG=${NEW_COMPUTE_TAG} docker compose --profile test-extensions up --quiet-pull --build -d
wait_for_ready wait_for_ready
docker compose exec neon-test-extensions psql -c "DROP DATABASE IF EXISTS contrib_regression" docker compose exec neon-test-extensions psql -c "DROP DATABASE IF EXISTS contrib_regression"
docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression"
@@ -55,7 +55,7 @@ create_extensions "${EXTNAMES}"
query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /\',\'}')" query="select json_object_agg(extname,extversion) from pg_extension where extname in ('${EXTNAMES// /\',\'}')"
new_vers=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query") new_vers=$(docker compose exec neon-test-extensions psql -Aqt -d contrib_regression -c "$query")
docker compose --profile test-extensions down docker compose --profile test-extensions down
TAG=${OLDTAG} docker compose --profile test-extensions up --quiet-pull --build -d --force-recreate COMPUTE_TAG=${OLD_COMPUTE_TAG} docker compose --profile test-extensions up --quiet-pull --build -d --force-recreate
wait_for_ready wait_for_ready
docker compose exec neon-test-extensions psql -c "DROP DATABASE IF EXISTS contrib_regression" docker compose exec neon-test-extensions psql -c "DROP DATABASE IF EXISTS contrib_regression"
docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression" docker compose exec neon-test-extensions psql -c "CREATE DATABASE contrib_regression"
@@ -86,8 +86,8 @@ else
) )
result=$(curl "${PARAMS[@]}") result=$(curl "${PARAMS[@]}")
echo $result | jq . echo $result | jq .
TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} TAG=${OLDTAG} docker compose down compute compute_is_ready TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} COMPUTE_TAG=${OLD_COMPUTE_TAG} docker compose down compute compute_is_ready
COMPUTE_TAG=${NEWTAG} TAG=${OLDTAG} TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} docker compose up --quiet-pull -d --build compute compute_is_ready COMPUTE_TAG=${NEW_COMPUTE_TAG} TENANT_ID=${tenant_id} TIMELINE_ID=${new_timeline_id} docker compose up --quiet-pull -d --build compute compute_is_ready
wait_for_ready wait_for_ready
TID=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id") TID=$(docker compose exec neon-test-extensions psql -Aqt -c "SHOW neon.timeline_id")
if [ ${TID} != ${new_timeline_id} ]; then if [ ${TID} != ${new_timeline_id} ]; then