windmill/.github/workflows/docker-image.yml

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.event_name !=
    'workflow_dispatch' && github.repository || 'windmill-labs/windmill-test' }}
  DEV_SHA: ${{ github.event_name != 'pull_request' && github.event_name !=
    'workflow_dispatch' && 'dev' || github.event.inputs.tag || github.sha }}
name: Build windmill:main
on:
  push:
    branches: [main]
    tags: ["*"]
  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - "Dockerfile"
  workflow_dispatch:
    inputs:
      ee:
        description: "Build EE image (true, false)"
        required: false
        default: false
        type: boolean
      tag:
        description: "Tag the image"
        required: true
        default: "test"
      slim:
        description: "Build slim image (true, false)"
        required: false
        default: false
        type: boolean
concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: false

permissions: write-all

jobs:
  build:
    runs-on: ubicloud
    if: (github.event_name != 'workflow_dispatch') || (github.event.inputs &&
      !github.event.inputs.ee)
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.ref }}
          fetch-depth: 0

      - name: Read EE repo commit hash
        run: |
          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          repository: windmill-labs/windmill-ee-private
          path: ./windmill-ee-private
          ref: ${{ env.ee_repo_ref }}
          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2
      - uses: depot/setup-action@v1

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Substitute EE code (EE logic is behind feature flag)
        run: |
          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private

      - name: Docker meta
        id: meta-public
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          flavor: |
            latest=false
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Build and push publicly
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          build-args: |
            features=ce
            WM_BUILD_VERSION=${{ github.sha }}
          tags: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
            ${{ steps.meta-public.outputs.tags }}
          labels: |
            ${{ steps.meta-public.outputs.labels }}

  build_ee:
    runs-on: ubicloud
    if: (github.event_name != 'workflow_dispatch') || github.event.inputs.ee
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.ref }}
          fetch-depth: 0

      - name: Read EE repo commit hash
        run: |
          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          repository: windmill-labs/windmill-ee-private
          path: ./windmill-ee-private
          ref: ${{ env.ee_repo_ref }}
          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2
      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee
          flavor: |
            latest=false
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Substitute EE code
        run: |
          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          build-args: |
            features=ee
            WM_BUILD_VERSION=${{ github.sha }}
          tags: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  attach_amd64_binary_to_release:
    needs: [build, build_ee]
    runs-on: ubicloud
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    env:
      ARCH: amd64
    steps:
      - uses: actions/checkout@v4
      - run: |
          # pulling docker image with desired arch so that actions-docker-extract doesn't do it
          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
          docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}

      - run: |
          # Checks the image is in docker prior to running actions-docker-extract. It fails if not
          # Also useful to visually check that the arch is the right opencontainers
          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
          docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}

      - uses: shrink/actions-docker-extract@v3
        id: extract
        with:
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
          path: "/usr/src/app/windmill"

      - uses: shrink/actions-docker-extract@v3
        id: extract-duckdb-ffi-internal
        with:
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
          path: "/usr/src/app/libwindmill_duckdb_ffi_internal.so"

      - uses: shrink/actions-docker-extract@v3
        id: extract-ee
        with:
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
          path: "/usr/src/app/windmill"

      - name: Rename binary with corresponding architecture
        run: |
          mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
          mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"

      - name: Attach binary to release
        uses: softprops/action-gh-release@v2
        with:
          files: |
            ${{ steps.extract.outputs.destination }}/*
            ${{ steps.extract-ee.outputs.destination }}/*
            ${{ steps.extract-duckdb-ffi-internal.outputs.destination }}/*

  attach_ee_debug_to_release:
    needs: [build_ee]
    runs-on: ubicloud
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    strategy:
      matrix:
        platform: [linux/amd64, linux/arm64]
        include:
          - platform: linux/amd64
            arch: amd64
          - platform: linux/arm64
            arch: arm64
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.ref }}

      - name: Read EE repo commit hash
        run: |
          echo "ee_repo_ref=$(cat ./backend/ee-repo-ref.txt)" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          repository: windmill-labs/windmill-ee-private
          path: ./windmill-ee-private
          ref: ${{ env.ee_repo_ref }}
          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}

      - name: Substitute EE code
        run: |
          ./backend/substitute_ee_code.sh --copy --dir ./windmill-ee-private

      - uses: depot/setup-action@v1

      - name: Extract EE debug info from builder stage (depot cache hit)
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: ${{ matrix.platform }}
          target: debuginfo
          build-args: |
            features=ee
            WM_BUILD_VERSION=${{ github.sha }}
          outputs: type=local,dest=./debuginfo

      - name: Rename debug file with corresponding architecture
        run: |
          mv ./debuginfo/windmill.debug ./debuginfo/windmill-ee-${{ matrix.arch }}.debug

      - name: Attach debug file to release
        uses: softprops/action-gh-release@v2
        with:
          files: ./debuginfo/windmill-ee-${{ matrix.arch }}.debug

  # attach_arm64_binary_to_release:
  #   needs: [build, build_ee]
  #   runs-on: ubicoud
  #   if: ${{ startsWith(github.ref, 'refs/tags/') }}
  #   env:
  #     ARCH: arm64
  #   steps:
  #     - uses: actions/checkout@v4

  #     - run: |
  #         # pulling docker image with desired arch so that actions-docker-extract doesn't do it
  #         docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
  #         docker pull --platform "linux/$ARCH" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}

  #     - run: |
  #         # Checks the image is in docker prior to running actions-docker-extract. It fails if not
  #         # Also useful to visually check that the arch is the right opencontainers
  #         docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
  #         docker image inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}

  #     - uses: shrink/actions-docker-extract@v3
  #       id: extract
  #       with:
  #         image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
  #         path: "/usr/src/app/windmill"

  #     - uses: shrink/actions-docker-extract@v3
  #       id: extract-ee
  #       with:
  #         image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
  #         path: "/usr/src/app/windmill"

  #     - name: Rename binary with corresponding architecture
  #       run: |
  #         mv "${{ steps.extract.outputs.destination }}/windmill" "${{ steps.extract.outputs.destination }}/windmill-${ARCH}"
  #         mv "${{ steps.extract-ee.outputs.destination }}/windmill" "${{ steps.extract-ee.outputs.destination }}/windmill-ee-${ARCH}"

  #     - name: Attach binary to release
  #       uses: softprops/action-gh-release@v2
  #       with:
  #         files: |
  #           ${{ steps.extract.outputs.destination }}/*
  #           ${{ steps.extract-ee.outputs.destination }}/*

  run_integration_test:
    runs-on: ubicloud
    needs: [build_ee]
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.ref }}
          fetch-depth: 0
      - name: Prepare test run
        if: ${{ ! startsWith(github.ref, 'refs/tags/v') }}
        run: cd integration_tests && ./build.sh
      - name: Test run
        if: ${{ ! startsWith(github.ref, 'refs/tags/v') }}
        timeout-minutes: 15
        env:
          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
        run: cd integration_tests && ./run.sh
      - name: Archive logs
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: Windmill Integration Tests Logs
          path: |
            integration_tests/logs

  tag_latest:
    runs-on: ubicloud
    needs: [run_integration_test, build]
    if:
      github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' ||
      startsWith(github.ref, 'refs/tags/v'))  && (github.event_name != 'workflow_dispatch')
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Tag main and latest
        run: |
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main

  tag_latest_ee:
    runs-on: ubicloud
    needs: [run_integration_test, build_ee]
    if:
      github.event_name != 'pull_request' && (github.event_name != 'workflow_dispatch') && (github.ref == 'refs/heads/main' ||
      startsWith(github.ref, 'refs/tags/v'))
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Tag main and latest for ee
        run: |
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:latest
          docker buildx imagetools create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} --tag ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main

  verify_ee_image_vulnerabilities:
    runs-on: ubicloud
    needs: [tag_latest_ee]
    if: startsWith(github.ref, 'refs/tags/v') && (github.event_name != 'workflow_dispatch')
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - name: Analyze for critical and high CVEs
        id: docker-scout-cves
        if: ${{ github.event_name != 'pull_request_target' }}
        uses: docker/scout-action@v1
        with:
          command: cves
          only-severities: critical,high
          image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:main
          sarif-file: sarif.output.json
          summary: true
          dockerhub-user: windmilllabs
          dockerhub-password: ${{ secrets.DOCKER_PAT }}

      - name: Upload SARIF result
        id: upload-sarif
        if: ${{ github.event_name != 'pull_request_target' }}
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: sarif.output.json

  # docker_scout_ee:
  #   runs-on: ubicloud
  #   needs: [tag_latest_ee]
  #   steps:
  #     - name: Docker Scout
  #       id: docker-scout
  #       uses: docker/scout-action@v1
  #       with:
  #         dockerhub-
  #         command: cves,recommendations,compare
  #         to-latest: true
  #         ignore-base: true
  #         ignore-unchanged: true
  #         only-fixed: true

  publish_ecr_s3:
    needs: [build_ee_full]
    runs-on: ubicloud-standard-2-arm
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Get version from tag
        id: version
        run: echo "VERSION=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"

      - uses: shrink/actions-docker-extract@v3
        id: extract
        with:
          image: |-
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full:${{ steps.version.outputs.VERSION }}
          path: "/static_frontend/."

      - uses: reggionick/s3-deploy@v4
        with:
          folder: ${{ steps.extract.outputs.destination }}
          bucket: windmill-frontend
          bucket-region: us-east-1

  build_ee_cuda:
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    needs: [build_ee]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.ref }}

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-cuda
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64
          push: true
          file: "./docker/DockerfileCuda"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  build_slim:
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    needs: [build]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-slim
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64
          push: true
          file: "./docker/DockerfileSlim"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}

  build_ee_slim:
    needs: [build_ee]
    runs-on: ubicloud
    if: (github.event_name != 'pull_request') && ((github.event_name != 'workflow_dispatch') || (github.event.inputs.ee || github.event.inputs.slim))

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-slim
          tags: |
            type=ref,event=branch
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          file: "./docker/DockerfileSlimEe"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License

  build_full:
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    needs: [build]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-public
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-full
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          file: "./docker/DockerfileFull"
          tags: |
            ${{ steps.meta-public.outputs.tags }}
          labels: |
            ${{ steps.meta-public.outputs.labels }}

  build_ee_full:
    if: ${{ startsWith(github.ref, 'refs/tags/v') }}
    needs: [build_ee]
    runs-on: ubicloud
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      # - name: Set up Docker Buildx
      #   uses: docker/setup-buildx-action@v2

      - uses: depot/setup-action@v1

      - name: Docker meta
        id: meta-ee-public
        uses: docker/metadata-action@v5
        with:
          images: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee-full
          tags: |
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}

      - name: Login to registry
        uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push publicly ee
        uses: depot/build-push-action@v1
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          file: "./docker/DockerfileFullEe"
          tags: |
            ${{ steps.meta-ee-public.outputs.tags }}
          labels: |
            ${{ steps.meta-ee-public.outputs.labels }}
            org.opencontainers.image.licenses=Windmill-Enterprise-License