perf: cut release CI wall time and build windmill-extra on its own release (#12329)

* perf: cut release CI wall time and build windmill-extra on its own release

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: wait for the ee-slim built from the release commit

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-11-10 18:18:32 +01:00
committed by GitHub
co-authored by Claude Opus 5.5
parent a9f5a9caa0
commit 7ec64ab33a
5 changed files with 76 additions and 41 deletions
+8 -23
View File
@@ -49,7 +49,10 @@ jobs:
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
# GIT_VERSION is `git describe --tags`, which needs the history and
# tags but no file contents beyond the checked-out tree
fetch-depth: 0
filter: blob:none
- name: Read EE repo commit hash
run: |
@@ -61,7 +64,6 @@ jobs:
path: ./windmill-ee-private
ref: ${{ env.ee_repo_ref }}
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
@@ -123,7 +125,10 @@ jobs:
- uses: actions/checkout@v4
with:
ref: ${{ github.ref }}
# GIT_VERSION is `git describe --tags`, which needs the history and
# tags but no file contents beyond the checked-out tree
fetch-depth: 0
filter: blob:none
- name: Read EE repo commit hash
run: |
@@ -135,7 +140,6 @@ jobs:
path: ./windmill-ee-private
ref: ${{ env.ee_repo_ref }}
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
@@ -371,8 +375,6 @@ jobs:
startsWith(github.ref, 'refs/tags/v')) && (github.event_name != 'workflow_dispatch')
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Login to registry
uses: docker/login-action@v3
with:
@@ -407,8 +409,6 @@ jobs:
startsWith(github.ref, 'refs/tags/v'))
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Login to registry
uses: docker/login-action@v3
with:
@@ -570,7 +570,7 @@ jobs:
# only-fixed: true
publish_ecr_s3:
needs: [build_ee_full]
needs: [build_ee]
runs-on: ubicloud-standard-2-arm
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
env:
@@ -578,8 +578,6 @@ jobs:
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
@@ -591,15 +589,11 @@ jobs:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get version from tag
id: version
run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
- uses: shrink/actions-docker-extract@v3
id: extract
with:
image: |-
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full:${{ steps.version.outputs.VERSION }}
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }}
path: "/static_frontend/."
- uses: reggionick/s3-deploy@v4
@@ -615,7 +609,6 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.ref }}
# - name: Set up Docker Buildx
@@ -672,8 +665,6 @@ jobs:
digest: ${{ steps.docker_build.outputs.digest }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
@@ -729,8 +720,6 @@ jobs:
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
@@ -785,8 +774,6 @@ jobs:
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
@@ -839,8 +826,6 @@ jobs:
runs-on: ubicloud
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# - name: Set up Docker Buildx
# uses: docker/setup-buildx-action@v2
+46 -10
View File
@@ -13,22 +13,56 @@ on:
permissions: write-all
jobs:
sleep:
# A release builds on its own ee-slim, pinned by digest (:latest is still the
# previous release while this one builds), and waits for its python client to
# be on PyPI, since lsp/Pipfile requires it.
base_image:
runs-on: ubicloud
outputs:
image: ${{ steps.base.outputs.image }}
steps:
- name: Sleep for 900 seconds waiting for pypi to update index
if: startsWith(github.ref, 'refs/tags/v')
run: sleep 900
- name: Wait for this release's ee-slim image and PyPI client
id: base
shell: bash
env:
IMAGE: ${{ env.REGISTRY }}/${{ github.repository }}-ee-slim
run: |
set -euo pipefail
if [[ "$GITHUB_REF" != refs/tags/v* ]]; then
echo "image=$IMAGE:latest" >> "$GITHUB_OUTPUT"
exit 0
fi
version="${GITHUB_REF_NAME#v}"
deadline=$((SECONDS + 75 * 60))
on_pypi() {
local page
page=$(curl -fsS -H 'Cache-Control: max-age=0' https://pypi.org/simple/wmill/) || return 1
grep -qF -e "wmill-${version}-py" -e "wmill-${version}.tar.gz" <<<"$page"
}
until on_pypi; do
(( SECONDS < deadline )) || { echo "wmill $version is not on PyPI"; exit 1; }
sleep 20
done
echo "wmill $version is on PyPI"
# A re-pushed tag finds the previous attempt's image under the same tag,
# so wait for the one built from this commit.
revision() {
docker buildx imagetools inspect "$IMAGE:$version" --format \
'{{ index (index .Image "linux/amd64").Config.Labels "org.opencontainers.image.revision" }}' 2>/dev/null
}
until [[ "$(revision)" == "$GITHUB_SHA" ]]; do
(( SECONDS < deadline )) || { echo "$IMAGE:$version for $GITHUB_SHA was never pushed"; exit 1; }
sleep 30
done
digest=$(docker buildx imagetools inspect "$IMAGE:$version" --format '{{.Manifest.Digest}}')
echo "image=$IMAGE@$digest" | tee -a "$GITHUB_OUTPUT"
# Build and test the image before publishing
test_extra:
needs: [sleep]
needs: [base_image]
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Setup Bun
uses: oven-sh/setup-bun@v2
@@ -62,6 +96,8 @@ jobs:
context: .
file: ./docker/DockerfileExtra
load: true
build-args: |
BASE_IMAGE=${{ needs.base_image.outputs.image }}
tags: windmill-extra:test
cache-from: type=gha
cache-to: type=gha,mode=max
@@ -102,15 +138,13 @@ jobs:
docker rm windmill-extra-test || true
publish_extra:
needs: [sleep, test_extra]
needs: [base_image, test_extra]
# a tag-targeted dispatch would republish the release tags unsigned,
# un-verifying the release; to republish a release, re-push its tag
if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/')
runs-on: ubicloud-standard-8
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- uses: depot/setup-action@v1
@@ -141,6 +175,8 @@ jobs:
file: ./docker/DockerfileExtra
platforms: linux/amd64,linux/arm64
push: true
build-args: |
BASE_IMAGE=${{ needs.base_image.outputs.image }}
sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }}
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+14 -4
View File
@@ -19,7 +19,10 @@
# ============================================================================
# Build final extra services image from windmill-ee-slim (includes nsjail)
# ============================================================================
FROM ghcr.io/windmill-labs/windmill-ee-slim:latest AS final
# Release builds pass this release's ee-slim by digest: at tag time :latest is
# still the previous release.
ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee-slim:latest
FROM ${BASE_IMAGE} AS final
ARG APP=/usr/src/app
@@ -138,9 +141,16 @@ RUN addgroup --gid 1000 windmill && \
# (/tmp/windmill/cache/uv) after the base already made it world-writable, leaving
# root-owned 0755 dirs that make uv fail EACCES for a non-root UID. /pyls/.cache
# (XDG_CACHE_HOME, incl. DENO_DIR) and /tmp/monaco are written at runtime too.
RUN chmod -R a+rX /usr/local /pyls /debugger /multiplayer /tmp/monaco && \
chmod -R a+rw /tmp/windmill /pyls/.cache && \
find /tmp/windmill /pyls/.cache /tmp/monaco -type d -exec chmod 777 {} +
# Each find touches only paths missing the bits (same result as chmod -R): a
# chmod on a file from an earlier layer copies it into this layer, so a blanket
# chmod -R costs 15 min and 330 MB per arch here.
RUN find /usr/local /pyls /debugger /multiplayer /tmp/monaco ! -type l \
\( ! -perm -a+r -o -perm /111 ! -perm -a+x -o -type d ! -perm -a+x \) \
-exec chmod a+rX {} + && \
find /tmp/windmill /pyls/.cache ! -type d ! -type l ! -perm -a+rw \
-exec chmod a+rw {} + && \
find /tmp/windmill /pyls/.cache /tmp/monaco -type d ! -perm 777 \
-exec chmod 777 {} +
# Expose all service ports
EXPOSE 3000 3001 3002 3003
+4 -2
View File
@@ -70,5 +70,7 @@ RUN apt-get install -y r-base-dev \
# Fix UV cache permissions for non-root user support (uid 1000, etc.)
# The uv tool install ansible command populates the UV cache with root-owned files
RUN chmod -R a+rw /tmp/windmill/cache/uv && \
find /tmp/windmill/cache/uv -type d -exec chmod 777 {} +
# Only paths missing the bits are touched: a chmod on a file from an earlier layer
# copies it into this layer, so a blanket chmod -R costs minutes per arch.
RUN find /tmp/windmill/cache/uv \( -type d ! -perm 777 -exec chmod 777 {} + \) \
-o \( ! -type d ! -type l ! -perm -a+rw -exec chmod a+rw {} + \)
+4 -2
View File
@@ -104,5 +104,7 @@ RUN apt-get install -y libsasl2-modules-gssapi-mit krb5-user
# Fix UV cache permissions for non-root user support (uid 1000, etc.)
# The uv tool install ansible command populates the UV cache with root-owned files
RUN chmod -R a+rw /tmp/windmill/cache/uv && \
find /tmp/windmill/cache/uv -type d -exec chmod 777 {} +
# Only paths missing the bits are touched: a chmod on a file from an earlier layer
# copies it into this layer, so a blanket chmod -R costs minutes per arch.
RUN find /tmp/windmill/cache/uv \( -type d ! -perm 777 -exec chmod 777 {} + \) \
-o \( ! -type d ! -type l ! -perm -a+rw -exec chmod a+rw {} + \)