windmill/.github/workflows/ai-agent-tests.yml

name: AI Agent Integration Tests

# Exercises the AI agent flow path (preview_flow with `aiagent` modules) against
# real LLM providers. Runs only when AI-agent backend code or the tests change,
# because each run makes real (paid) LLM calls. To avoid spending on every commit,
# the PR side triggers only when a PR is marked ready for review (out of draft) —
# not on `synchronize` — plus push to main and manual dispatch.
on:
  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - "integration_tests/ai_agent_tests/**"
      - "backend/windmill-ai/**"
      - "backend/windmill-api/src/ai.rs"
      - "backend/windmill-worker/src/ai_executor.rs"
      - "backend/windmill-worker/src/ai/**"
      - "backend/windmill-worker/src/memory_common.rs"
      - "backend/windmill-common/src/flow_conversations.rs"
      - ".github/workflows/ai-agent-tests.yml"
  pull_request:
    types: [opened, reopened, ready_for_review]
    paths:
      - "integration_tests/ai_agent_tests/**"
      - "backend/windmill-ai/**"
      - "backend/windmill-api/src/ai.rs"
      - "backend/windmill-worker/src/ai_executor.rs"
      - "backend/windmill-worker/src/ai/**"
      - "backend/windmill-worker/src/memory_common.rs"
      - "backend/windmill-common/src/flow_conversations.rs"
      - ".github/workflows/ai-agent-tests.yml"

concurrency:
  group: ai-agent-tests-${{ github.ref }}
  cancel-in-progress: true

jobs:
  ai_agent_e2e:
    # Skip draft PRs; the `opened`/`reopened` types would otherwise fire while
    # still a draft. `ready_for_review` always arrives non-draft.
    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
    runs-on: ubicloud-standard-16
    services:
      postgres:
        image: postgres:16
        ports:
          - 5432:5432
        env:
          POSTGRES_DB: windmill
          POSTGRES_PASSWORD: changeme
        options: >-
          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
          --health-retries 5
    steps:
      - uses: actions/checkout@v4

      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache-workspaces: backend
          toolchain: 1.97.0

      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: 1.3.10

      - uses: actions/setup-node@v4
        with:
          node-version: "20"

      - uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      # CE build (no enterprise/license needed for AI agents). `quickjs` powers
      # flow input-transform JS eval; `mcp` is required by the deepwiki MCP tool
      # test. Bun tool scripts run via the always-on worker (BUN_PATH).
      - name: Build Windmill
        working-directory: ./backend
        env:
          SQLX_OFFLINE: true
          CARGO_BUILD_JOBS: 12
          RUSTFLAGS: ""
        run: cargo build --features quickjs,mcp

      - name: Start Windmill
        working-directory: ./backend
        env:
          DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill
          BUN_PATH: bun
          NODE_BIN_PATH: node
          RUST_LOG: info
        run: |
          mkdir -p ../integration_tests/logs
          ./target/debug/windmill > ../integration_tests/logs/windmill.log 2>&1 &
          echo "Waiting for Windmill to be ready..."
          for i in $(seq 1 60); do
            if curl -sf http://localhost:8000/api/version > /dev/null 2>&1; then
              echo "Windmill is ready"
              break
            fi
            sleep 2
          done
          curl -sf http://localhost:8000/api/version > /dev/null || { echo "Windmill failed to start"; tail -50 ../integration_tests/logs/windmill.log; exit 1; }

      - name: Run AI agent integration tests
        timeout-minutes: 20
        working-directory: ./integration_tests/ai_agent_tests
        env:
          WINDMILL_URL: http://localhost:8000
          # Only the providers we have org secrets for. Other providers
          # (Azure, Bedrock, OpenRouter) are skipped by conftest when their
          # keys are absent — see skip_provider_without_credentials.
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GOOGLE_AI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
        run: |
          python -m venv .venv
          .venv/bin/pip install -r requirements.txt
          # The S3/vision-attachment tests need MinIO large-file storage and
          # image-capable provider setup; out of scope for this cost-controlled
          # smoke. Add MinIO secrets + a storage service to enable them.
          .venv/bin/python -m pytest -v \
            --ignore=test_user_attachments.py \
            --ignore=test_user_images.py \
            --ignore=test_image_output.py

      - name: Archive Windmill logs
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: ai-agent-tests-windmill-logs
          path: integration_tests/logs