From 7ec64ab33acc56870975c05f24a983bdcd146ee2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Tue, 10 Nov 2026 17:18:32 +0000 Subject: [PATCH] perf: cut release CI wall time and build windmill-extra on its own release (#12329) * perf: cut release CI wall time and build windmill-extra on its own release Co-Authored-By: Claude Opus 5.5 * fix: wait for the ee-slim built from the release commit Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/workflows/docker-image.yml | 31 +++++----------- .github/workflows/publish_extra.yml | 56 +++++++++++++++++++++++------ docker/DockerfileExtra | 18 +++++++--- docker/DockerfileFull | 6 ++-- docker/DockerfileFullEe | 6 ++-- 5 files changed, 76 insertions(+), 41 deletions(-) diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index a5823ef19c..4ef7adc46f 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -49,7 +49,10 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.ref }} + # GIT_VERSION is `git describe --tags`, which needs the history and + # tags but no file contents beyond the checked-out tree fetch-depth: 0 + filter: blob:none - name: Read EE repo commit hash run: | @@ -61,7 +64,6 @@ jobs: path: ./windmill-ee-private ref: ${{ env.ee_repo_ref }} token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} - fetch-depth: 0 # - name: Set up Docker Buildx # uses: docker/setup-buildx-action@v2 @@ -123,7 +125,10 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.ref }} + # GIT_VERSION is `git describe --tags`, which needs the history and + # tags but no file contents beyond the checked-out tree fetch-depth: 0 + filter: blob:none - name: Read EE repo commit hash run: | @@ -135,7 +140,6 @@ jobs: path: ./windmill-ee-private ref: ${{ env.ee_repo_ref }} token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }} - fetch-depth: 0 # - name: Set up Docker Buildx # uses: docker/setup-buildx-action@v2 @@ -371,8 +375,6 @@ jobs: startsWith(github.ref, 'refs/tags/v')) && (github.event_name != 'workflow_dispatch') steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Login to registry uses: docker/login-action@v3 with: @@ -407,8 +409,6 @@ jobs: startsWith(github.ref, 'refs/tags/v')) steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Login to registry uses: docker/login-action@v3 with: @@ -570,7 +570,7 @@ jobs: # only-fixed: true publish_ecr_s3: - needs: [build_ee_full] + needs: [build_ee] runs-on: ubicloud-standard-2-arm if: ${{ startsWith(github.ref, 'refs/tags/v') }} env: @@ -578,8 +578,6 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -591,15 +589,11 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Get version from tag - id: version - run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT" - - uses: shrink/actions-docker-extract@v3 id: extract with: image: |- - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full:${{ steps.version.outputs.VERSION }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee@${{ needs.build_ee.outputs.digest }} path: "/static_frontend/." - uses: reggionick/s3-deploy@v4 @@ -615,7 +609,6 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 ref: ${{ github.ref }} # - name: Set up Docker Buildx @@ -672,8 +665,6 @@ jobs: digest: ${{ steps.docker_build.outputs.digest }} steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 # - name: Set up Docker Buildx # uses: docker/setup-buildx-action@v2 @@ -729,8 +720,6 @@ jobs: steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 # - name: Set up Docker Buildx # uses: docker/setup-buildx-action@v2 @@ -785,8 +774,6 @@ jobs: runs-on: ubicloud steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 # - name: Set up Docker Buildx # uses: docker/setup-buildx-action@v2 @@ -839,8 +826,6 @@ jobs: runs-on: ubicloud steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 # - name: Set up Docker Buildx # uses: docker/setup-buildx-action@v2 diff --git a/.github/workflows/publish_extra.yml b/.github/workflows/publish_extra.yml index c1db4a654d..4f9ec08e1e 100644 --- a/.github/workflows/publish_extra.yml +++ b/.github/workflows/publish_extra.yml @@ -13,22 +13,56 @@ on: permissions: write-all jobs: - sleep: + # A release builds on its own ee-slim, pinned by digest (:latest is still the + # previous release while this one builds), and waits for its python client to + # be on PyPI, since lsp/Pipfile requires it. + base_image: runs-on: ubicloud + outputs: + image: ${{ steps.base.outputs.image }} steps: - - name: Sleep for 900 seconds waiting for pypi to update index - if: startsWith(github.ref, 'refs/tags/v') - run: sleep 900 + - name: Wait for this release's ee-slim image and PyPI client + id: base shell: bash + env: + IMAGE: ${{ env.REGISTRY }}/${{ github.repository }}-ee-slim + run: | + set -euo pipefail + if [[ "$GITHUB_REF" != refs/tags/v* ]]; then + echo "image=$IMAGE:latest" >> "$GITHUB_OUTPUT" + exit 0 + fi + version="${GITHUB_REF_NAME#v}" + deadline=$((SECONDS + 75 * 60)) + on_pypi() { + local page + page=$(curl -fsS -H 'Cache-Control: max-age=0' https://pypi.org/simple/wmill/) || return 1 + grep -qF -e "wmill-${version}-py" -e "wmill-${version}.tar.gz" <<<"$page" + } + until on_pypi; do + (( SECONDS < deadline )) || { echo "wmill $version is not on PyPI"; exit 1; } + sleep 20 + done + echo "wmill $version is on PyPI" + # A re-pushed tag finds the previous attempt's image under the same tag, + # so wait for the one built from this commit. + revision() { + docker buildx imagetools inspect "$IMAGE:$version" --format \ + '{{ index (index .Image "linux/amd64").Config.Labels "org.opencontainers.image.revision" }}' 2>/dev/null + } + until [[ "$(revision)" == "$GITHUB_SHA" ]]; do + (( SECONDS < deadline )) || { echo "$IMAGE:$version for $GITHUB_SHA was never pushed"; exit 1; } + sleep 30 + done + digest=$(docker buildx imagetools inspect "$IMAGE:$version" --format '{{.Manifest.Digest}}') + echo "image=$IMAGE@$digest" | tee -a "$GITHUB_OUTPUT" # Build and test the image before publishing test_extra: - needs: [sleep] + needs: [base_image] runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Setup Bun uses: oven-sh/setup-bun@v2 @@ -62,6 +96,8 @@ jobs: context: . file: ./docker/DockerfileExtra load: true + build-args: | + BASE_IMAGE=${{ needs.base_image.outputs.image }} tags: windmill-extra:test cache-from: type=gha cache-to: type=gha,mode=max @@ -102,15 +138,13 @@ jobs: docker rm windmill-extra-test || true publish_extra: - needs: [sleep, test_extra] + needs: [base_image, test_extra] # a tag-targeted dispatch would republish the release tags unsigned, # un-verifying the release; to republish a release, re-push its tag if: github.event_name == 'push' || !startsWith(github.ref, 'refs/tags/') runs-on: ubicloud-standard-8 steps: - uses: actions/checkout@v4 - with: - fetch-depth: 0 - uses: depot/setup-action@v1 @@ -141,6 +175,8 @@ jobs: file: ./docker/DockerfileExtra platforms: linux/amd64,linux/arm64 push: true + build-args: | + BASE_IMAGE=${{ needs.base_image.outputs.image }} sbom: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name == 'push' }} tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest diff --git a/docker/DockerfileExtra b/docker/DockerfileExtra index 7eb2de6bf7..2c79221cf7 100644 --- a/docker/DockerfileExtra +++ b/docker/DockerfileExtra @@ -19,7 +19,10 @@ # ============================================================================ # Build final extra services image from windmill-ee-slim (includes nsjail) # ============================================================================ -FROM ghcr.io/windmill-labs/windmill-ee-slim:latest AS final +# Release builds pass this release's ee-slim by digest: at tag time :latest is +# still the previous release. +ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill-ee-slim:latest +FROM ${BASE_IMAGE} AS final ARG APP=/usr/src/app @@ -138,9 +141,16 @@ RUN addgroup --gid 1000 windmill && \ # (/tmp/windmill/cache/uv) after the base already made it world-writable, leaving # root-owned 0755 dirs that make uv fail EACCES for a non-root UID. /pyls/.cache # (XDG_CACHE_HOME, incl. DENO_DIR) and /tmp/monaco are written at runtime too. -RUN chmod -R a+rX /usr/local /pyls /debugger /multiplayer /tmp/monaco && \ - chmod -R a+rw /tmp/windmill /pyls/.cache && \ - find /tmp/windmill /pyls/.cache /tmp/monaco -type d -exec chmod 777 {} + +# Each find touches only paths missing the bits (same result as chmod -R): a +# chmod on a file from an earlier layer copies it into this layer, so a blanket +# chmod -R costs 15 min and 330 MB per arch here. +RUN find /usr/local /pyls /debugger /multiplayer /tmp/monaco ! -type l \ + \( ! -perm -a+r -o -perm /111 ! -perm -a+x -o -type d ! -perm -a+x \) \ + -exec chmod a+rX {} + && \ + find /tmp/windmill /pyls/.cache ! -type d ! -type l ! -perm -a+rw \ + -exec chmod a+rw {} + && \ + find /tmp/windmill /pyls/.cache /tmp/monaco -type d ! -perm 777 \ + -exec chmod 777 {} + # Expose all service ports EXPOSE 3000 3001 3002 3003 diff --git a/docker/DockerfileFull b/docker/DockerfileFull index d145a5ffff..e1299de610 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -70,5 +70,7 @@ RUN apt-get install -y r-base-dev \ # Fix UV cache permissions for non-root user support (uid 1000, etc.) # The uv tool install ansible command populates the UV cache with root-owned files -RUN chmod -R a+rw /tmp/windmill/cache/uv && \ - find /tmp/windmill/cache/uv -type d -exec chmod 777 {} + +# Only paths missing the bits are touched: a chmod on a file from an earlier layer +# copies it into this layer, so a blanket chmod -R costs minutes per arch. +RUN find /tmp/windmill/cache/uv \( -type d ! -perm 777 -exec chmod 777 {} + \) \ + -o \( ! -type d ! -type l ! -perm -a+rw -exec chmod a+rw {} + \) diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 8ca1067f7a..3b9817d96d 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -104,5 +104,7 @@ RUN apt-get install -y libsasl2-modules-gssapi-mit krb5-user # Fix UV cache permissions for non-root user support (uid 1000, etc.) # The uv tool install ansible command populates the UV cache with root-owned files -RUN chmod -R a+rw /tmp/windmill/cache/uv && \ - find /tmp/windmill/cache/uv -type d -exec chmod 777 {} + +# Only paths missing the bits are touched: a chmod on a file from an earlier layer +# copies it into this layer, so a blanket chmod -R costs minutes per arch. +RUN find /tmp/windmill/cache/uv \( -type d ! -perm 777 -exec chmod 777 {} + \) \ + -o \( ! -type d ! -type l ! -perm -a+rw -exec chmod a+rw {} + \)