windmill/.github/workflows/backend-test.yml

name: Backend only integration tests

on:
  workflow_dispatch:
  push:
    branches:
      - "main"
    paths:
      - "backend/**"
      - ".github/workflows/backend-test.yml"
  pull_request:
    types: [opened, synchronize, reopened]
    paths:
      - "backend/**"
      - ".github/workflows/backend-test.yml"

defaults:
  run:
    working-directory: ./backend

jobs:
  cargo_test:
    runs-on: ubicloud-standard-16
    services:
      postgres:
        image: postgres
        ports:
          - 5432:5432
        env:
          POSTGRES_DB: windmill
          POSTGRES_PASSWORD: changeme
          POSTGRES_INITDB_ARGS: "-c max_connections=500"
        options: >-
          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
          --health-retries 5 --shm-size=256mb
      mysql:
        image: mysql:8.0
        ports:
          - 3306:3306
        env:
          MYSQL_ROOT_PASSWORD: changeme
          MYSQL_DATABASE: windmill_test
        options: >-
          --health-cmd "mysqladmin ping -h localhost" --health-interval 10s
          --health-timeout 5s --health-retries 5
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-dotnet@v4
        with:
          dotnet-version: "9.0.x"
      - uses: denoland/setup-deno@v2
        with:
          # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI
          # tests what production runs, instead of floating on the latest v2.x.
          deno-version: 2.2.1
      - uses: actions/setup-go@v2
        with:
          go-version: 1.21.5
      - uses: oven-sh/setup-bun@v2
        with:
          bun-version: 1.3.10
      - uses: actions/setup-node@v4
        with:
          node-version: "20"
      - uses: astral-sh/setup-uv@v6.2.1
        with:
          version: "0.11.24"
      - uses: shivammathur/setup-php@v2
        with:
          php-version: "8.3"
          tools: composer
      - uses: ruby/setup-ruby@v1
        with:
          ruby-version: "3.3"
          bundler-cache: false
      - name: Install windmill CLI from source
        run: |
          cd $GITHUB_WORKSPACE/cli
          bash gen_wm_client.sh
          bun install
          mkdir -p "$HOME/.local/bin"
          printf '#!/bin/sh\nexec bun run "%s/cli/src/main.ts" "$@"\n' "$GITHUB_WORKSPACE" > "$HOME/.local/bin/wmill"
          chmod +x "$HOME/.local/bin/wmill"
          echo "$HOME/.local/bin" >> $GITHUB_PATH
        working-directory: /
      - name: Install PowerShell, mold and clang
        run: |
          sudo apt-get update && sudo apt-get install -y powershell mold clang libcurl4-openssl-dev
        working-directory: /
      - uses: actions-rust-lang/setup-rust-toolchain@v1
        with:
          cache-workspaces: backend
          toolchain: 1.97.0
      - name: Fix stale v8 build cache
        working-directory: ./backend
        run: |
          # Cargo cache may preserve v8 build fingerprints without the actual
          # librusty_v8.a library. Since fingerprints look valid, cargo skips
          # build.rs re-run, causing "could not find native static library rusty_v8".
          for profile in debug release; do
            if [ -d "target/$profile/.fingerprint" ] && [ ! -f "target/$profile/gn_out/obj/librusty_v8.a" ]; then
              echo "Cleaning stale v8 build artifacts in target/$profile"
              rm -rf "target/$profile/build/v8-"* "target/$profile/.fingerprint/v8-"*
            fi
          done
      - name: Read EE repo commit hash
        run: |
          echo "ee_repo_ref=$(cat ./ee-repo-ref.txt)" >> "$GITHUB_ENV"

      - uses: actions/checkout@v4
        with:
          repository: windmill-labs/windmill-ee-private
          path: ./windmill-ee-private
          ref: ${{ env.ee_repo_ref }}
          token: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
          fetch-depth: 0

      - name: Substitute EE code (EE logic is behind feature flag)
        run: |
          ./substitute_ee_code.sh --copy --dir ./windmill-ee-private
      - name: Setup private npm registry with test package
        working-directory: /tmp
        run: |
          set -e

          # Install Verdaccio globally
          npm install -g verdaccio

          # Create Verdaccio config that requires authentication for @windmill-test packages
          mkdir -p /tmp/verdaccio/storage
          cat > /tmp/verdaccio/config.yaml << 'VERDACCIO_CONFIG'
          storage: /tmp/verdaccio/storage
          auth:
            htpasswd:
              file: /tmp/verdaccio/htpasswd
              max_users: 100
          uplinks:
            npmjs:
              url: https://registry.npmjs.org/
          packages:
            '@windmill-test/*':
              access: $authenticated
              publish: $authenticated
            '@*/*':
              access: $all
              publish: $authenticated
              proxy: npmjs
            '**':
              access: $all
              publish: $authenticated
              proxy: npmjs
          server:
            keepAliveTimeout: 60
          middlewares:
            audit:
              enabled: true
          log: { type: stdout, format: pretty, level: warn }
          VERDACCIO_CONFIG

          # Create empty htpasswd file (users will be created via API)
          touch /tmp/verdaccio/htpasswd

          # Start Verdaccio in background
          verdaccio --config /tmp/verdaccio/config.yaml &
          VERDACCIO_PID=$!

          # Wait for Verdaccio to be ready
          echo "Waiting for Verdaccio to start..."
          for i in {1..30}; do
            if curl -s http://localhost:4873/-/ping > /dev/null 2>&1; then
              echo "Verdaccio is ready"
              break
            fi
            sleep 1
          done

          # Login to get a token
          echo "Getting auth token..."
          RESPONSE=$(curl -s -X PUT \
            -H "Content-Type: application/json" \
            -d '{"name":"testuser","password":"testpass123"}' \
            http://localhost:4873/-/user/org.couchdb.user:testuser)

          echo "Auth response: $RESPONSE"
          NPM_TOKEN=$(echo "$RESPONSE" | jq -r '.token')

          if [ -z "$NPM_TOKEN" ] || [ "$NPM_TOKEN" = "null" ]; then
            echo "Failed to get NPM token from response"
            exit 1
          fi

          echo "NPM_TOKEN=${NPM_TOKEN}" >> $GITHUB_ENV
          {
            echo "TEST_NPMRC<<NPMRC_EOF"
            echo "@windmill-test:registry=http://localhost:4873/"
            echo "//localhost:4873/:_authToken=${NPM_TOKEN}"
            echo "NPMRC_EOF"
          } >> $GITHUB_ENV
          echo "Got NPM token successfully: ${NPM_TOKEN:0:10}..."

          # Configure npm globally with the auth token
          echo "//localhost:4873/:_authToken=${NPM_TOKEN}" > ~/.npmrc
          echo "Configured ~/.npmrc with auth token"

          # Create a simple test package
          mkdir -p /tmp/windmill-test-private-pkg
          cat > /tmp/windmill-test-private-pkg/package.json << 'PKG_JSON'
          {
            "name": "@windmill-test/private-pkg",
            "version": "1.0.0",
            "main": "index.js"
          }
          PKG_JSON
          cat > /tmp/windmill-test-private-pkg/index.js << 'PKG_JS'
          module.exports.greet = (name) => `Hello from private package, ${name}!`;
          PKG_JS

          # Publish to Verdaccio with auth
          cd /tmp/windmill-test-private-pkg
          echo "Publishing package..."
          npm publish --registry http://localhost:4873
          echo "Package published successfully"

          # Verify the package requires auth by trying anonymous access (should fail)
          rm -f ~/.npmrc
          echo "Testing anonymous access (should fail)..."
          if npm view @windmill-test/private-pkg --registry http://localhost:4873 2>/dev/null; then
            echo "ERROR: Package should require authentication but anonymous access worked"
            exit 1
          fi
          echo "Verified: Package requires authentication for @windmill-test/private-pkg"
      - name: Cache DuckDB FFI module build
        uses: actions/cache@v3
        with:
          path: ./backend/windmill-duckdb-ffi-internal/target
          key: ${{ runner.os }}-duckdb-ffi-${{ hashFiles('./backend/windmill-duckdb-ffi-internal/src/**/*.rs', './backend/windmill-duckdb-ffi-internal/Cargo.toml', './backend/windmill-duckdb-ffi-internal/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-duckdb-ffi-
      - name: cargo test
        timeout-minutes: 30
        env:
          # setup-rust-toolchain exports RUSTFLAGS=-D warnings, and the RUSTFLAGS env
          # var fully REPLACES (never merges with) target.*.rustflags in
          # backend/.cargo/config.toml. That silently drops the config's
          # `-C link-arg=-fuse-ld=mold`, so CI links the many large integration-test
          # binaries (v8 + duckdb + every language runtime, statically linked) with the
          # default bfd linker. Its peak memory across ~12 parallel links OOM-kills the
          # runner mid-link (SIGTERM => exit 143, before any test runs). Re-add the mold
          # link arg here so CI links with mold like local dev, keeping -D warnings.
          # (config.toml's `linker = "clang"` still applies; env only overrides rustflags.)
          RUSTFLAGS: "-D warnings -C link-arg=-fuse-ld=mold"
          SQLX_OFFLINE: true
          DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill
          DISABLE_EMBEDDING: true
          RUST_LOG: "off"
          RUST_LOG_STYLE: never
          CARGO_NET_GIT_FETCH_WITH_CLI: true
          # Cap parallel rustc/link jobs below the 16 available cores. The tail of
          # the build links ~128 full-graph test binaries (one per tests/*.rs file
          # across the workspace); at high parallelism enough heavy codegen+link
          # units (rustc ~2.6GB, mold ~1GB each) overlap to exhaust the 64GB
          # runner. Matches backend-test-windows.yml, which already uses 8.
          CARGO_BUILD_JOBS: 8
          # Incremental compilation is per-run dead weight in CI: rust-cache
          # (cache-workspaces above) restores compiled dependency artifacts but
          # never persists target/**/incremental, so there is no prior state to
          # reuse in a one-shot `cargo test`. It only adds per-crate memory
          # overhead and extra disk. Off here (kept on for local dev via
          # .cargo/config.toml). Matches backend-test-windows.yml.
          CARGO_INCREMENTAL: "0"
          # backend/Cargo.toml leaves profile.dev at the default debug = 2 for
          # the (large) windmill workspace crates; that debug info is emitted
          # into every object file and embedded in each test binary. Across the
          # full --all --features build it is the dominant memory/disk consumer
          # when mold links the windmill-api-integration-tests binary, tipping
          # the runner over (lost runner reported as a canceled step). CI needs
          # no debug info, so drop it entirely for the dev/test profiles here.
          # (test profile inherits dev, but the workspace crates link in as
          # dev-profile deps, so both must be set.) CI-only; local dev builds
          # are unaffected.
          CARGO_PROFILE_DEV_DEBUG: "0"
          CARGO_PROFILE_TEST_DEBUG: "0"
          # Tests' poll-time stack frames (deep nested async fn chains in
          # debug builds) reach ~1.8MB, leaving very thin headroom on the
          # default 2MB thread stack. 4MB gives ~2x buffer against flaky
          # overflows under parallel-test contention.
          RUST_MIN_STACK: 4194304
          WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: 1
          WMDEBUG_FORCE_RUNNABLE_SETTINGS_V0: 1
          WMDEBUG_FORCE_NO_LEGACY_DEBOUNCING_COMPAT: 1
          TEST_NPM_REGISTRY: "http://localhost:4873/:_authToken=${{ env.NPM_TOKEN }}"
        run: |
          deno --version && bun -v && node --version && go version && python3 --version && php --version && ruby --version && pwsh --version && dotnet --version
          # The FFI crate is excluded from the workspace, so the `cargo test` below
          # never reaches it. Pin the target dir (matching the cache step above) so
          # its own tests run off this compile rather than a second bundled build.
          (cd windmill-duckdb-ffi-internal && export CARGO_TARGET_DIR="$PWD/target" && ./build_dev.sh && cargo test --release -p windmill_duckdb_ffi_internal)
          DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline --all -- --nocapture --test-threads=10