mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
perf: cut release CI wall time and build windmill-extra on its own release (#12329)
* perf: cut release CI wall time and build windmill-extra on its own release Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: wait for the ee-slim built from the release commit Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a9f5a9caa0
commit
7ec64ab33a
@@ -49,7 +49,10 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
# GIT_VERSION is `git describe --tags`, which needs the history and
|
||||
# tags but no file contents beyond the checked-out tree
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- name: Read EE repo commit hash
|
||||
run: |
|
||||
@@ -61,7 +64,6 @@ jobs:
|
||||
path: ./windmill-ee-private
|
||||
ref: ${{ env.ee_repo_ref }}
|
||||
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
|
||||
fetch-depth: 0
|
||||
|
||||
# - name: Set up Docker Buildx
|
||||
# uses: docker/setup-buildx-action@v2
|
||||
@@ -123,7 +125,10 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref }}
|
||||
# GIT_VERSION is `git describe --tags`, which needs the history and
|
||||
# tags but no file contents beyond the checked-out tree
|
||||
fetch-depth: 0
|
||||
filter: blob:none
|
||||
|
||||
- name: Read EE repo commit hash
|
||||
run: |
|
||||
@@ -135,7 +140,6 @@ jobs:
|
||||
path: ./windmill-ee-private
|
||||
ref: ${{ env.ee_repo_ref }}
|
||||
token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
|
||||
fetch-depth: 0
|
||||
|
||||
# - name: Set up Docker Buildx
|
||||
# uses: docker/setup-buildx-action@v2
|
||||
@@ -371,8 +375,6 @@ jobs:
|
||||
startsWith(github.ref, 'refs/tags/v')) && (github.event_name != 'workflow_dispatch')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Login to registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -407,8 +409,6 @@ jobs:
|
||||
startsWith(github.ref, 'refs/tags/v'))
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- name: Login to registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
@@ -570,7 +570,7 @@ jobs:
|
||||
# only-fixed: true
|
||||
|
||||
publish_ecr_s3:
|
||||
needs: [build_ee_full]
|
||||
needs: [build_ee]
|
||||
runs-on: ubicloud-standard-2-arm
|
||||
if: ${{ startsWith(github.ref, 'refs/tags/v') }}
|
||||
env:
|
||||
@@ -578,8 +578,6 @@ jobs:
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
@@ -591,15 +589,11 @@ jobs:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Get version from tag
|
||||
id: version
|
||||
run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: shrink/actions-docker-extract@v3
|
||||
id: extract
|
||||
with:
|
||||
image: |-
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full:${{ steps.version.outputs.VERSION }}
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }}
|
||||
path: "/static_frontend/."
|
||||
|
||||
- uses: reggionick/s3-deploy@v4
|
||||
@@ -615,7 +609,6 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.ref }}
|
||||
|
||||
# - name: Set up Docker Buildx
|
||||
@@ -672,8 +665,6 @@ jobs:
|
||||
digest: ${{ steps.docker_build.outputs.digest }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# - name: Set up Docker Buildx
|
||||
# uses: docker/setup-buildx-action@v2
|
||||
@@ -729,8 +720,6 @@ jobs:
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# - name: Set up Docker Buildx
|
||||
# uses: docker/setup-buildx-action@v2
|
||||
@@ -785,8 +774,6 @@ jobs:
|
||||
runs-on: ubicloud
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# - name: Set up Docker Buildx
|
||||
# uses: docker/setup-buildx-action@v2
|
||||
@@ -839,8 +826,6 @@ jobs:
|
||||
runs-on: ubicloud
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# - name: Set up Docker Buildx
|
||||
# uses: docker/setup-buildx-action@v2
|
||||
|
||||
@@ -13,22 +13,56 @@ on:
|
||||
permissions: write-all
|
||||
|
||||
jobs:
|
||||
sleep:
|
||||
# A release builds on its own ee-slim, pinned by digest (:latest is still the
|
||||
# previous release while this one builds), and waits for its python client to
|
||||
# be on PyPI, since lsp/Pipfile requires it.
|
||||
base_image:
|
||||
runs-on: ubicloud
|
||||
outputs:
|
||||
image: ${{ steps.base.outputs.image }}
|
||||
steps:
|
||||
- name: Sleep for 900 seconds waiting for pypi to update index
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
run: sleep 900
|
||||
- name: Wait for this release's ee-slim image and PyPI client
|
||||
id: base
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ env.REGISTRY }}/${{ github.repository }}-ee-slim
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "$GITHUB_REF" != refs/tags/v* ]]; then
|
||||
echo "image=$IMAGE:latest" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
version="${GITHUB_REF_NAME#v}"
|
||||
deadline=$((SECONDS + 75 * 60))
|
||||
on_pypi() {
|
||||
local page
|
||||
page=$(curl -fsS -H 'Cache-Control: max-age=0' https://pypi.org/simple/wmill/) || return 1
|
||||
grep -qF -e "wmill-${version}-py" -e "wmill-${version}.tar.gz" <<<"$page"
|
||||
}
|
||||
until on_pypi; do
|
||||
(( SECONDS < deadline )) || { echo "wmill $version is not on PyPI"; exit 1; }
|
||||
sleep 20
|
||||
done
|
||||
echo "wmill $version is on PyPI"
|
||||
# A re-pushed tag finds the previous attempt's image under the same tag,
|
||||
# so wait for the one built from this commit.
|
||||
revision() {
|
||||
docker buildx imagetools inspect "$IMAGE:$version" --format \
|
||||
'{{ index (index .Image "linux/amd64").Config.Labels "org.opencontainers.image.revision" }}' 2>/dev/null
|
||||
}
|
||||
until [[ "$(revision)" == "$GITHUB_SHA" ]]; do
|
||||
(( SECONDS < deadline )) || { echo "$IMAGE:$version for $GITHUB_SHA was never pushed"; exit 1; }
|
||||
sleep 30
|
||||
done
|
||||
digest=$(docker buildx imagetools inspect "$IMAGE:$version" --format '{{.Manifest.Digest}}')
|
||||
echo "image=$IMAGE@$digest" | tee -a "$GITHUB_OUTPUT"
|
||||
|
||||
# Build and test the image before publishing
|
||||
test_extra:
|
||||
needs: [sleep]
|
||||
needs: [base_image]
|
||||
runs-on: ubicloud-standard-8
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
@@ -62,6 +96,8 @@ jobs:
|
||||
context: .
|
||||
file: ./docker/DockerfileExtra
|
||||
load: true
|
||||
build-args: |
|
||||
BASE_IMAGE=${{ needs.base_image.outputs.image }}
|
||||
tags: windmill-extra:test
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
@@ -102,15 +138,13 @@ jobs:
|
||||
docker rm windmill-extra-test || true
|
||||
|
||||
publish_extra:
|
||||
needs: [sleep, test_extra]
|
||||
needs: [base_image, test_extra]
|
||||
# a tag-targeted dispatch would republish the release tags unsigned,
|
||||
# un-verifying the release; to republish a release, re-push its tag
|
||||
if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/')
|
||||
runs-on: ubicloud-standard-8
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: depot/setup-action@v1
|
||||
|
||||
@@ -141,6 +175,8 @@ jobs:
|
||||
file: ./docker/DockerfileExtra
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: true
|
||||
build-args: |
|
||||
BASE_IMAGE=${{ needs.base_image.outputs.image }}
|
||||
sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }}
|
||||
tags: |
|
||||
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
|
||||
|
||||
+14
-4
@@ -19,7 +19,10 @@
|
||||
# ============================================================================
|
||||
# Build final extra services image from windmill-ee-slim (includes nsjail)
|
||||
# ============================================================================
|
||||
FROM ghcr.io/windmill-labs/windmill-ee-slim:latest AS final
|
||||
# Release builds pass this release's ee-slim by digest: at tag time :latest is
|
||||
# still the previous release.
|
||||
ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee-slim:latest
|
||||
FROM ${BASE_IMAGE} AS final
|
||||
|
||||
ARG APP=/usr/src/app
|
||||
|
||||
@@ -138,9 +141,16 @@ RUN addgroup --gid 1000 windmill && \
|
||||
# (/tmp/windmill/cache/uv) after the base already made it world-writable, leaving
|
||||
# root-owned 0755 dirs that make uv fail EACCES for a non-root UID. /pyls/.cache
|
||||
# (XDG_CACHE_HOME, incl. DENO_DIR) and /tmp/monaco are written at runtime too.
|
||||
RUN chmod -R a+rX /usr/local /pyls /debugger /multiplayer /tmp/monaco && \
|
||||
chmod -R a+rw /tmp/windmill /pyls/.cache && \
|
||||
find /tmp/windmill /pyls/.cache /tmp/monaco -type d -exec chmod 777 {} +
|
||||
# Each find touches only paths missing the bits (same result as chmod -R): a
|
||||
# chmod on a file from an earlier layer copies it into this layer, so a blanket
|
||||
# chmod -R costs 15 min and 330 MB per arch here.
|
||||
RUN find /usr/local /pyls /debugger /multiplayer /tmp/monaco ! -type l \
|
||||
\( ! -perm -a+r -o -perm /111 ! -perm -a+x -o -type d ! -perm -a+x \) \
|
||||
-exec chmod a+rX {} + && \
|
||||
find /tmp/windmill /pyls/.cache ! -type d ! -type l ! -perm -a+rw \
|
||||
-exec chmod a+rw {} + && \
|
||||
find /tmp/windmill /pyls/.cache /tmp/monaco -type d ! -perm 777 \
|
||||
-exec chmod 777 {} +
|
||||
|
||||
# Expose all service ports
|
||||
EXPOSE 3000 3001 3002 3003
|
||||
|
||||
@@ -70,5 +70,7 @@ RUN apt-get install -y r-base-dev \
|
||||
|
||||
# Fix UV cache permissions for non-root user support (uid 1000, etc.)
|
||||
# The uv tool install ansible command populates the UV cache with root-owned files
|
||||
RUN chmod -R a+rw /tmp/windmill/cache/uv && \
|
||||
find /tmp/windmill/cache/uv -type d -exec chmod 777 {} +
|
||||
# Only paths missing the bits are touched: a chmod on a file from an earlier layer
|
||||
# copies it into this layer, so a blanket chmod -R costs minutes per arch.
|
||||
RUN find /tmp/windmill/cache/uv \( -type d ! -perm 777 -exec chmod 777 {} + \) \
|
||||
-o \( ! -type d ! -type l ! -perm -a+rw -exec chmod a+rw {} + \)
|
||||
|
||||
@@ -104,5 +104,7 @@ RUN apt-get install -y libsasl2-modules-gssapi-mit krb5-user
|
||||
|
||||
# Fix UV cache permissions for non-root user support (uid 1000, etc.)
|
||||
# The uv tool install ansible command populates the UV cache with root-owned files
|
||||
RUN chmod -R a+rw /tmp/windmill/cache/uv && \
|
||||
find /tmp/windmill/cache/uv -type d -exec chmod 777 {} +
|
||||
# Only paths missing the bits are touched: a chmod on a file from an earlier layer
|
||||
# copies it into this layer, so a blanket chmod -R costs minutes per arch.
|
||||
RUN find /tmp/windmill/cache/uv \( -type d ! -perm 777 -exec chmod 777 {} + \) \
|
||||
-o \( ! -type d ! -type l ! -perm -a+rw -exec chmod a+rw {} + \)
|
||||
|
||||
Reference in New Issue
Block a user