diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml new file mode 100644 index 0000000000..964e5d0cab --- /dev/null +++ b/.github/workflows/ai-evals-scheduled.yml @@ -0,0 +1,171 @@ +name: AI Evals (scheduled) + +# Full ai_evals suites on current models, posted to the AI evals dashboard in +# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked +# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M +# tokens per model: every suite runs on MODELS, and global (the mode users get) +# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by +# hand to measure a branch against main. +on: + schedule: + - cron: "0 3 * * 1" + workflow_dispatch: + inputs: + modes: + description: "Space-separated modes" + default: "global flow app script cli" + models: + description: "Space-separated model aliases (bun run cli -- models)" + default: "sonnet-5.5" + global_extra_models: + description: "Extra model aliases for global mode only" + default: "gpt-6-astra gemini-3.8-flash" + runs: + description: "Runs per case" + default: "3" + reasoning: + description: "Reasoning effort for frontend modes (empty: the product default)" + default: "" + +concurrency: + group: ai-evals-scheduled-${{ github.ref }} + +env: + MODELS: ${{ inputs.models || 'sonnet-5.5' }} + GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }} + RUNS: ${{ inputs.runs || '3' }} + REASONING: ${{ inputs.reasoning }} + INGEST_URL: https://app.windmill.dev/api/r/f/ai/ingest_ai_eval_run + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + modes: ${{ steps.modes.outputs.modes }} + steps: + - id: modes + env: + MODES: ${{ inputs.modes || 'global flow app script cli' }} + run: | + echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ") + | map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT" + + evals: + needs: setup + runs-on: ubicloud-standard-16 + timeout-minutes: 330 + strategy: + fail-fast: false + matrix: + mode: ${{ fromJSON(needs.setup.outputs.modes) }} + services: + postgres: + image: postgres:16 + ports: + - 5432:5432 + env: + POSTGRES_DB: windmill + POSTGRES_PASSWORD: changeme + options: >- + --health-cmd pg_isready --health-interval 10s --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@v4 + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache-workspaces: backend + toolchain: 1.97.0 + + - uses: oven-sh/setup-bun@v2 + with: + bun-version: 1.4.0 + + - uses: actions/setup-node@v7 + with: + node-version: "24" + + - name: Build Windmill + working-directory: ./backend + env: + SQLX_OFFLINE: true + CARGO_BUILD_JOBS: 12 + RUSTFLAGS: "" + run: cargo build --features quickjs + + - name: Start Windmill + working-directory: ./backend + env: + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill + RUST_LOG: info + run: | + mkdir -p ../ai_evals/logs + ./target/debug/windmill > ../ai_evals/logs/windmill.log 2>&1 & + for i in $(seq 1 60); do + curl -sf http://localhost:8000/api/version > /dev/null 2>&1 && break + sleep 2 + done + curl -sf http://localhost:8000/api/version > /dev/null || { tail -50 ../ai_evals/logs/windmill.log; exit 1; } + + - name: Install frontend deps + generate client + working-directory: ./frontend + run: | + npm ci + npm run generate-backend-client + + - name: Install CLI deps + generate CLI client + working-directory: ./cli + run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh + + - name: Run ${{ matrix.mode }} evals + working-directory: ./ai_evals + env: + MODE: ${{ matrix.mode }} + WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000 + WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + run: | + bun install + mkdir -p results + models="$MODELS" + [ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS" + reasoning=() + [ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING") + for m in $models; do + bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \ + --output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored" + done + + - name: Post results to the dashboard + if: always() + working-directory: ./ai_evals + env: + MODE: ${{ matrix.mode }} + INGEST_TOKEN: ${{ secrets.AI_EVALS_INGEST_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + [ -n "$INGEST_TOKEN" ] || { echo "::warning::AI_EVALS_INGEST_TOKEN is not set"; exit 0; } + shopt -s nullglob + for f in results/"$MODE"-*.json; do + # Keep what the dashboard reads; the full traces stay in the run's artifacts. + jq -c --arg ref "$GITHUB_REF" --arg trigger "$GITHUB_EVENT_NAME" --arg url "$RUN_URL" ' + {result_json: (del(.cases[].prompt, .cases[].initialPath, .cases[].expectedPath) + | .cases[].attempts[] |= {attempt, passed, durationMs, toolCallCount, judgeScore, + judgeSummary, error, tokenUsage, checks: [.checks[]? | {name, passed}]}), + git_ref: $ref, trigger: $trigger, run_url: $url}' "$f" \ + | curl -sf --retry 3 -X POST "$INGEST_URL" \ + -H "Authorization: Bearer $INGEST_TOKEN" -H 'content-type: application/json' \ + --data-binary @- && echo " <- $f" || echo "::warning::failed to post $f" + done + + - name: Archive logs and results + uses: actions/upload-artifact@v4 + if: always() + with: + name: ai-evals-${{ matrix.mode }} + path: | + ai_evals/logs + ai_evals/results diff --git a/ai_evals/README.md b/ai_evals/README.md index 603f28e5f9..bff4294e52 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -77,31 +77,20 @@ Public CLI surface: - `--verbose`: stream assistant output for frontend runs - `--skip-judge`: skip LLM judge scoring for the run - `--execution-only`: only require the model/proxy/frontend loop to complete; skip validators, tool expectations, backend artifact validation, and judge scoring +- `--reasoning `: reasoning effort for frontend modes (`off`, `low`, `medium`, `high`, `max`, …); without it the product's default applies (`high` on models that can reason). The effort is appended to the recorded model label (`anthropic:claude-sonnet-5-5@max`) - `--record`: append a compact tracked summary line to `ai_evals/history/.jsonl` for full-suite runs only - `--backend-validation `: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals ## Models -Use `bun run cli -- models` to see the current aliases. - -Today: - -- `haiku` -- `sonnet` -- `opus` -- `4o` -- `gpt-5.5` -- `gemini-3-flash-preview` -- `gemini-3.1-pro-preview` -- `deepseek-v4-flash` -- `deepseek-v4-pro` +Use `bun run cli -- models` to see the current aliases; `core/models.ts` is the list. Notes: - the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5` - frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases - `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there -- the judge model is separate and currently defaults to `claude-sonnet-4-6`; use `--skip-judge` for deterministic-only runs +- the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs ## Case Format diff --git a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts index b9c80f3554..0ff352f3d4 100644 --- a/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts +++ b/ai_evals/adapters/frontend/core/script/scriptEvalRunner.ts @@ -15,6 +15,7 @@ import { runEval } from "../shared"; import type { ModeRunContext } from "../../../../core/types"; import type { TokenUsage, ToolCallDetail } from "../shared/types"; import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings"; +import { evalReasoningEffort } from "../shared/providerConfig"; export interface ScriptEvalResult { success: boolean; @@ -41,14 +42,15 @@ export interface ScriptEvalOptions { function resolveModelProvider( model: string, provider?: AIProvider, -): AIProviderModel { +): AIProviderModel & { reasoning?: string } { + const reasoning = evalReasoningEffort(); if (provider) { - return { provider, model }; + return { provider, model, reasoning }; } if (model.startsWith("claude")) { - return { provider: "anthropic", model }; + return { provider: "anthropic", model, reasoning }; } - return { provider: "openai", model }; + return { provider: "openai", model, reasoning }; } export async function runScriptEval( diff --git a/ai_evals/adapters/frontend/core/shared/providerConfig.ts b/ai_evals/adapters/frontend/core/shared/providerConfig.ts index 9d9e315217..49d5cb6ef8 100644 --- a/ai_evals/adapters/frontend/core/shared/providerConfig.ts +++ b/ai_evals/adapters/frontend/core/shared/providerConfig.ts @@ -12,6 +12,12 @@ export interface EvalClients { export interface ResolvedEvalModelProvider { provider: FrontendEvalProvider; model: string; + reasoning?: string; +} + +/** `run --reasoning` hands the effort to the frontend runtime through the environment. */ +export function evalReasoningEffort(): string | undefined { + return process.env.WMILL_AI_EVAL_REASONING || undefined; } export interface WindmillAiProxyClientConfig { @@ -73,6 +79,13 @@ export function createEvalClients(input: { export function resolveEvalModelProvider( model: string, provider?: FrontendEvalProvider, +): ResolvedEvalModelProvider { + return { ...resolveProvider(model, provider), reasoning: evalReasoningEffort() }; +} + +function resolveProvider( + model: string, + provider?: FrontendEvalProvider, ): ResolvedEvalModelProvider { if (provider) { return { provider, model }; diff --git a/ai_evals/bun.lock b/ai_evals/bun.lock index eaed1db99a..798022d49a 100644 --- a/ai_evals/bun.lock +++ b/ai_evals/bun.lock @@ -5,8 +5,8 @@ "": { "name": "windmill-ai-evals", "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.25", - "@anthropic-ai/sdk": "^0.39.0", + "@anthropic-ai/claude-agent-sdk": "^0.3.284", + "@anthropic-ai/sdk": "^0.129.0", "commander": "^14.0.3", "openai": "^6.9.1", "yaml": "^2.8.3", @@ -18,66 +18,44 @@ }, }, "packages": { - "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.87", "", { "dependencies": { "@anthropic-ai/sdk": "^0.74.0", "@modelcontextprotocol/sdk": "^1.27.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-WWmgBPxPhBOvNT0ujI8vPTI2lK+w5YEkEZ/y1mH0EDkK/0kBnxVJNhCtG5vnueiAViwLoUOFn66pbkDiivijdA=="], + "@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.3.284", "", { "optionalDependencies": { "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.284" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.0.0" } }, "sha512-NSoJwEq6nFSf8dtaacYx37QdGgqApI3eHFUlxclMLYi8irb6ZJwEaUnPnLUCyAyg9W/tMkgZC0GXWLRCc01I0w=="], - "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="], + "@anthropic-ai/claude-agent-sdk-darwin-arm64": ["@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.284", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gKY9MUjY83398uCiPLHsd87kyzu7agIM7ApqWpJkpSINep6hxx4rNoR8bUbNWg49Aoe/PW/DJkQByAQuNzR9rg=="], - "@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="], + "@anthropic-ai/claude-agent-sdk-darwin-x64": ["@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.284", "", { "os": "darwin", "cpu": "x64" }, "sha512-P+q6Z7sKeYz99uE7RJc4au1IK+4SiWe69pyHY3QXMnjZq8hSFU/5Mo8iSurLk9jRlihsMltV1rLA5naouYyCAQ=="], + + "@anthropic-ai/claude-agent-sdk-linux-arm64": ["@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-LDpuYDaz+pCdG29iy1pw4P1D2YVtpZOb4rWOPxxSpg2fFyUeQ5WrfnuszOxhDWpNh1k1ekDiKm3A9OzGcrkdtA=="], + + "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": ["@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-U3/uAv1TS8sMWsgWQSxYbdND5AaRXXpP5RxHU1oYC5ZizlZE8/xAVum9dj1QX3rVZJg8EzsqZveq/AkEv5ABfQ=="], + + "@anthropic-ai/claude-agent-sdk-linux-x64": ["@anthropic-ai/claude-agent-sdk-linux-x64@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-yGytBCCwJvWeg1FzFpgnLM9BOM2vKVExtv6pMJHMtF82J5yhKHcxodRaYlQVd/HzJJ2nruKb9EnrhRBZ//4yRw=="], + + "@anthropic-ai/claude-agent-sdk-linux-x64-musl": ["@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-4x0Q8CFkCNbeENE7pRhuUmkwh/sQfTq1yWi4ZE56stve9pgoKzu6ZDa1SEtvK1YIooGE/qM5rVS8xFLE1gbQ0w=="], + + "@anthropic-ai/claude-agent-sdk-win32-arm64": ["@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.284", "", { "os": "win32", "cpu": "arm64" }, "sha512-zOXkdPHhElyxyFJ6eY5R+YGUswqBDZ+ZHNaRmQGJ28RkQYgwG5imNy/nfFR+/CPGlMnH9ZzbGc2rKYApm1K4Tg=="], + + "@anthropic-ai/claude-agent-sdk-win32-x64": ["@anthropic-ai/claude-agent-sdk-win32-x64@0.3.284", "", { "os": "win32", "cpu": "x64" }, "sha512-VGaFRDCOPloj5IjvJTyy5JsSl9sE2HR3W+A6f/cjgh23/7Y5vK7/ka+1JQD+AkoGO9tbHqP2O6god3IBrLvOaw=="], + + "@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.129.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1", "standardwebhooks": "^1.0.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-MH7LB20kNpGLUpPTe2OG5XvL/KhX8HUO9XyBaFjgFk6TLS4Xjt0VPIuMKywe7POKKPslX+QxSlVok9e+8kG5Nw=="], + + "@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="], "@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="], - "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], - - "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], - - "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], - - "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], - - "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], - - "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], - - "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], - - "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], - - "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], - - "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], - - "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], - - "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], - - "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], - - "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], - - "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], - - "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], - "@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="], + "@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="], + "@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="], - "@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="], - - "@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="], - - "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], + "@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], - "agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="], - "ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="], "ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="], - "asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="], - "body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="], "bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="], @@ -88,8 +66,6 @@ "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], - "combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="], - "commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="], "content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="], @@ -106,8 +82,6 @@ "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], - "delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="], - "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], @@ -122,14 +96,10 @@ "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], - "es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="], - "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], - "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="], - "eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="], "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], @@ -140,16 +110,12 @@ "fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="], + "fast-sha256": ["fast-sha256@1.3.0", "", {}, "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ=="], + "fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="], "finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="], - "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], - - "form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="], - - "formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="], - "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], @@ -164,16 +130,12 @@ "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], - "has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="], - "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], "hono": ["hono@4.12.9", "", {}, "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA=="], "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], - "humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="], - "iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="], "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], @@ -208,10 +170,6 @@ "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], - "node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="], - - "node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="], - "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], @@ -262,30 +220,24 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "standardwebhooks": ["standardwebhooks@1.1.1", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ=="], + "statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="], "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], - "tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="], - "ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="], "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], - "undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="], + "undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], - "web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="], - - "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], - - "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], @@ -295,19 +247,5 @@ "zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="], - - "@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="], - - "@types/node-fetch/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], - - "bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="], - - "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - - "@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], - - "bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="], - - "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], } } diff --git a/ai_evals/cli/index.ts b/ai_evals/cli/index.ts index 504b8a8d13..8a5f63c8b2 100644 --- a/ai_evals/cli/index.ts +++ b/ai_evals/cli/index.ts @@ -108,6 +108,10 @@ async function main() { "--record", "append a compact summary line to ai_evals/history/.jsonl", ) + .option( + "--reasoning ", + "reasoning effort for frontend modes (e.g. off, low, medium, high, max); default: the product's", + ) .option( "--backend-validation ", `backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`, @@ -126,6 +130,7 @@ async function main() { executionOnly?: boolean; record?: boolean; backendValidation?: string; + reasoning?: string; }, ) => { await handleRun({ @@ -140,6 +145,7 @@ async function main() { executionOnly: options.executionOnly ?? false, record: options.record ?? false, backendValidation: options.backendValidation, + reasoning: options.reasoning, }); }, ); @@ -190,6 +196,7 @@ async function handleRun(input: { executionOnly: boolean; record: boolean; backendValidation?: string; + reasoning?: string; }) { if (input.record && input.caseIds.length > 0) { throw new Error( @@ -217,6 +224,13 @@ async function handleRun(input: { "--backend-validation currently supports only flow and script modes", ); } + if (input.reasoning) { + if (input.mode === "cli") { + throw new Error("--reasoning only applies to frontend modes"); + } + // The frontend runtime runs in a child process, which inherits it. + process.env.WMILL_AI_EVAL_REASONING = input.reasoning; + } if (input.mode !== "cli") { await assertWindmillBackendReachable(resolveWindmillBackendSettings()); } @@ -257,6 +271,10 @@ async function handleRun(input: { backendValidation, }); + if (input.reasoning && result.runModel) { + result.runModel = `${result.runModel}@${input.reasoning}`; + } + const resolvedOutputPath = models.length === 1 ? resolveRunOutputPath(input.mode, input.outputPath) diff --git a/ai_evals/core/judge.ts b/ai_evals/core/judge.ts index 5e67510027..f1df60f286 100644 --- a/ai_evals/core/judge.ts +++ b/ai_evals/core/judge.ts @@ -1,7 +1,7 @@ import Anthropic from "@anthropic-ai/sdk"; import type { EvalMode, JudgeResult } from "./types"; -export const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-6"; +export const DEFAULT_JUDGE_MODEL = "claude-sonnet-5-5"; const JUDGE_TOOL_NAME = "submit_judgement"; @@ -29,6 +29,7 @@ export async function judgeOutput(input: { const system = [ "You evaluate benchmark outputs for Windmill AI generation.", + `Always answer by calling the ${JUDGE_TOOL_NAME} tool.`, "Deterministic checks already run separately. Focus on whether the final output satisfies the user request.", "If expected state is provided, treat it as a valid example and reward semantically equivalent outputs.", "If a checklist is provided, treat it as the explicit acceptance criteria for this case.", @@ -69,8 +70,8 @@ export async function judgeOutput(input: { try { const response = await client.messages.create({ model, - max_tokens: 1024, - temperature: 0, + // The judge thinks by default, and thinking shares this budget with the verdict. + max_tokens: 16000, system, messages: [{ role: "user", content: user }], tools: [ @@ -93,11 +94,8 @@ export async function judgeOutput(input: { }, }, ], - tool_choice: { - type: "tool", - name: JUDGE_TOOL_NAME, - disable_parallel_tool_use: true, - }, + // Current models refuse a forced tool_choice ("tool"/"any"). + tool_choice: { type: "auto", disable_parallel_tool_use: true }, }); const toolUseBlock = response.content.find( diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 295cd36135..1580cf5b0b 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -78,6 +78,32 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "opus", }, }, + { + id: "sonnet-5.5", + label: "Claude Sonnet 5.5", + aliases: ["sonnet-5.5", "claude-sonnet-5.5", "claude-sonnet-5-5"], + frontend: { + provider: "anthropic", + model: "claude-sonnet-5-5", + }, + cli: { + provider: "anthropic", + model: "claude-sonnet-5-5", + }, + }, + { + id: "opus-5.5", + label: "Claude Opus 5.5", + aliases: ["opus-5.5", "claude-opus-5.5", "claude-opus-5-5"], + frontend: { + provider: "anthropic", + model: "claude-opus-5-5", + }, + cli: { + provider: "anthropic", + model: "claude-opus-5-5", + }, + }, { id: "4o", label: "GPT-4o", @@ -96,6 +122,51 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "gpt-5.5", }, }, + { + id: "gpt-5.6-sol", + label: "GPT-5.6 Sol", + aliases: ["gpt-5.6-sol"], + frontend: { + provider: "openai", + model: "gpt-5.6-sol", + }, + }, + { + id: "gpt-6-astra", + label: "GPT-6 Astra", + aliases: ["gpt-6-astra", "gpt-6"], + frontend: { + provider: "openai", + model: "gpt-6-astra", + }, + }, + { + id: "gpt-6-sol", + label: "GPT-6 Sol", + aliases: ["gpt-6-sol"], + frontend: { + provider: "openai", + model: "gpt-6-sol", + }, + }, + { + id: "gpt-6-luna", + label: "GPT-6 Luna", + aliases: ["gpt-6-luna"], + frontend: { + provider: "openai", + model: "gpt-6-luna", + }, + }, + { + id: "gemini-3.8-flash", + label: "Gemini 3.8 Flash", + aliases: ["gemini-3.8-flash"], + frontend: { + provider: "googleai", + model: "gemini-3.8-flash", + }, + }, { id: "gemini-3-flash-preview", label: "Gemini 3 Flash Preview", diff --git a/ai_evals/package.json b/ai_evals/package.json index 6720c910cb..424e49ffae 100644 --- a/ai_evals/package.json +++ b/ai_evals/package.json @@ -8,8 +8,8 @@ "test:frontend-graph": "cd ../frontend && node_modules/.bin/vitest run --project server --config ../ai_evals/adapters/frontend/vitest.unit.config.ts" }, "dependencies": { - "@anthropic-ai/claude-agent-sdk": "^0.2.25", - "@anthropic-ai/sdk": "^0.39.0", + "@anthropic-ai/claude-agent-sdk": "^0.3.284", + "@anthropic-ai/sdk": "^0.129.0", "commander": "^14.0.3", "openai": "^6.9.1", "yaml": "^2.8.3" diff --git a/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json b/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json new file mode 100644 index 0000000000..a7b61d7282 --- /dev/null +++ b/backend/.sqlx/query-0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH RECURSIVE descendants AS (\n SELECT id FROM v2_job WHERE parent_job = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id FROM v2_job j JOIN descendants d ON j.parent_job = d.id\n WHERE j.workspace_id = $2\n ) SELECT d.id AS \"id!\" FROM descendants d JOIN v2_job_queue q ON q.id = d.id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771" +} diff --git a/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json b/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json new file mode 100644 index 0000000000..8209f67c75 --- /dev/null +++ b/backend/.sqlx/query-451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET started_at = now() WHERE id = $1 RETURNING started_at", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a" +} diff --git a/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json b/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json deleted file mode 100644 index cc6a57c005..0000000000 --- a/backend/.sqlx/query-52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT c.id IS NOT NULL AS \"completed!\",\n c.result AS \"result: sqlx::types::Json>\",\n c.status = 'success' AS \"success\",\n EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1) AS \"queued!\"\n FROM (SELECT 1) one\n LEFT JOIN v2_job_completed c ON c.id = $1 AND c.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "completed!", - "type_info": "Bool" - }, - { - "ordinal": 1, - "name": "result: sqlx::types::Json>", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "success", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "queued!", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - null, - true, - null, - null - ] - }, - "hash": "52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5" -} diff --git a/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json b/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json new file mode 100644 index 0000000000..9702cbf61d --- /dev/null +++ b/backend/.sqlx/query-789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id, status = 'success' AS \"success!\", result AS \"result: Json>\"\n FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "success!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "result: Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "UuidArray" + ] + }, + "nullable": [ + false, + null, + true + ] + }, + "hash": "789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc" +} diff --git a/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json b/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json new file mode 100644 index 0000000000..197ecaf3d9 --- /dev/null +++ b/backend/.sqlx/query-f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue SET worker = $1, running = true, started_at = now()\n WHERE id = $2 AND tag = ANY($3)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c1cdb2326c..114fb76b67 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14876,6 +14876,7 @@ dependencies = [ "windmill-worker-volumes", "windows-service", "windows-sys 0.52.0", + "winresource", ] [[package]] @@ -17382,6 +17383,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winresource" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087" +dependencies = [ + "version_check", +] + [[package]] name = "winsafe" version = "0.0.19" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5970ada6d8..9c09265dbd 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -329,6 +329,9 @@ strum.workspace = true windows-service = "0.7" windows-sys = { version = "0.52", features = ["Win32_System_Services", "Win32_System_Console", "Win32_Foundation"] } +[target.'cfg(windows)'.build-dependencies] +winresource = { version = "0.1.31", default-features = false } + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } diff --git a/backend/build.rs b/backend/build.rs index d5068697c7..88917b0492 100644 --- a/backend/build.rs +++ b/backend/build.rs @@ -2,4 +2,19 @@ fn main() { // trigger recompilation when a new migration is added println!("cargo:rerun-if-changed=migrations"); + + // Embed a VERSIONINFO resource so the exe's file properties carry the release + // version (from Cargo.toml), which Windows worker update scripts read. + // `cfg(windows)` is the host, matching the Windows-only build-dependency, so + // cross builds to Windows get no version resource. + #[cfg(windows)] + if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") { + winresource::WindowsResource::new() + .set("ProductName", "Windmill") + .set("FileDescription", "Windmill") + .set("CompanyName", "Windmill Labs, Inc.") + .set("LegalCopyright", "Copyright Windmill Labs, Inc.") + .compile() + .expect("failed to embed the Windows version resource"); + } } diff --git a/backend/tests/ai_tool_queue.rs b/backend/tests/ai_tool_queue.rs new file mode 100644 index 0000000000..4d39176cc9 --- /dev/null +++ b/backend/tests/ai_tool_queue.rs @@ -0,0 +1,220 @@ +use axum::{ + extract::State, + routing::{get, post}, + Json, Router, +}; +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; +use std::{sync::Arc, time::Duration}; +use tokio::sync::Notify; +use windmill_common::{flows::FlowValue, jobs::JobPayload}; +use windmill_test_utils::{ + completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind, +}; + +async fn model(Json(body): Json) -> ([(&'static str, &'static str); 1], String) { + let finished = body["messages"] + .as_array() + .unwrap() + .iter() + .any(|m| m["role"] == "tool"); + let delta = if finished { + json!({"role":"assistant", "content":"done"}) + } else { + json!({"role":"assistant", "tool_calls": (0..3).map(|i| json!({ + "index":i, "id":format!("call_{i}"), "type":"function", + "function":{"name":format!("tool_{i}"), "arguments":"{}"} + })).collect::>()}) + }; + let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]}); + let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]}); + ( + [("content-type", "text/event-stream")], + format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"), + ) +} + +async fn run_batch( + db: Pool, + parallel: bool, + limited: bool, + unsupported: bool, +) -> anyhow::Result<()> { + std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true"); + let server = ApiServer::start(db.clone()).await?; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let base = format!("http://{}", listener.local_addr()?); + let gate = Arc::new(Notify::new()); + let router = Router::new() + .route("/v1/chat/completions", post(model)) + .route( + "/first", + get(|State(gate): State>| async move { + gate.notified().await; + "ok" + }), + ) + .route( + "/second", + get(|State(gate): State>| async move { + gate.notify_one(); + "ok" + }), + ) + .with_state(gate); + let stub = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + let tools: Vec = (0..3).map(|i| { + let wait = if parallel && !limited && i < 2 { + format!("await fetch('{base}/{}');", if i == 0 {"first"} else {"second"}) + } else { String::new() }; + let finish = if i == 2 { "throw new Error('expected tool failure');".to_string() } + else { format!("return {i};") }; + json!({"id":format!("t{i}"),"summary":format!("tool_{i}"),"value":{ + "type":"rawscript", "language":"bun", "input_transforms":{}, + "tag": if unsupported && i != 1 { "unserved-tool-tag" } else { "bun" }, + "concurrent_limit": if limited { Some(1) } else { None }, + "custom_concurrency_key": if limited { Some("agent-tool-test") } else { None }, + "content":format!("export async function main() {{ {wait} await Bun.sleep({}); {finish} }}", if i == 0 {200} else {0}) + }}) + }).collect(); + let flow: FlowValue = serde_json::from_value( + json!({"modules":[{"id":"agent","timeout":if unsupported {Some(3)} else {None},"value":{ + "type":"aiagent", "tools":tools, "input_transforms":{ + "provider":{"type":"static","value":{"kind":"customai","model":"queue-test","resource":{"base_url":format!("{base}/v1")}}}, + "user_message":{"type":"static","value":"run the tools"}, + "max_iterations":{"type":"static","value":3} + } + }}]}), + )?; + let id = RunJob::from(JobPayload::RawFlow { + value: flow, + path: Some("u/test/agent_queue".into()), + restarted_from: None, + }) + .push(&db) + .await; + let notifications = listen_for_completed_jobs(&db).await; + let wait = async { + if parallel { + in_test_worker(&db, notifications.find(&id), server.addr.port()).await; + } else { + notifications.find(&id).await; + } + }; + tokio::time::timeout( + Duration::from_secs(45), + in_test_worker(&db, wait, server.addr.port()), + ) + .await?; + let result = completed_job(id, &db).await; + if unsupported { + assert!( + !result.success, + "agent must time out waiting for unsupported tools" + ); + let children: Vec<(String, bool, bool)> = sqlx::query_as( + "SELECT j.runnable_path, c.status = 'success', c.started_at IS NULL + FROM v2_job j JOIN v2_job_completed c USING(id) + WHERE j.parent_job IN (SELECT id FROM v2_job WHERE parent_job = $1) + ORDER BY j.runnable_path", + ) + .bind(id) + .fetch_all(&db) + .await?; + assert_eq!(children.len(), 3); + assert_eq!( + children.iter().map(|c| (c.1, c.2)).collect::>(), + vec![(false, true), (true, false), (false, true)], + "neither the first-tool reservation nor fallback may execute unsupported tags" + ); + stub.abort(); + server.close().await?; + return Ok(()); + } + assert!(result.success, "{:?}", result.result); + let result = result.json_result().expect("agent result"); + let messages: Vec<_> = result["messages"] + .as_array() + .unwrap() + .iter() + .filter(|m| m["role"] == "tool") + .collect(); + assert_eq!(messages.len(), 3); + for (index, message) in messages.iter().enumerate() { + assert_eq!(message["tool_call_id"], format!("call_{index}")); + } + assert_eq!(messages[0]["content"], "0"); + assert_eq!(messages[1]["content"], "1"); + assert!(messages[2]["content"] + .as_str() + .unwrap() + .contains("expected tool failure")); + let (parent_id, parent_worker): (uuid::Uuid, String) = sqlx::query_as( + "SELECT j.id, c.worker FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1" + ).bind(id).fetch_one(&db).await?; + let children: Vec<(String, String, bool)> = sqlx::query_as( + "SELECT j.runnable_path, c.worker, c.status = 'success' FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1 ORDER BY j.runnable_path" + ).bind(parent_id).fetch_all(&db).await?; + assert_eq!(children.len(), 3); + assert_eq!( + children[0].1, parent_worker, + "first tool must stay on the parent worker" + ); + assert_eq!( + children.iter().map(|c| c.2).collect::>(), + vec![true, true, false] + ); + if limited { + let overlapping: i64 = sqlx::query_scalar( + "SELECT count(*) FROM v2_job j1 JOIN v2_job_completed c1 ON c1.id = j1.id + JOIN v2_job j2 ON j2.parent_job = j1.parent_job AND j2.id > j1.id + JOIN v2_job_completed c2 ON c2.id = j2.id + WHERE j1.parent_job = $1 AND c1.started_at < c2.completed_at AND c2.started_at < c1.completed_at" + ).bind(parent_id).fetch_one(&db).await?; + assert_eq!( + overlapping, 0, + "the reserved first job must count toward the shared limit" + ); + } else if parallel { + assert_ne!( + children[1].1, parent_worker, + "second tool must unblock the first from another worker" + ); + } else { + assert!(children.iter().all(|child| child.1 == parent_worker)); + } + stub.abort(); + server.close().await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_drains_tools_without_another_worker(db: Pool) -> anyhow::Result<()> { + run_batch(db, false, false, false).await +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_keeps_first_tool_while_another_worker_runs_siblings( + db: Pool, +) -> anyhow::Result<()> { + run_batch(db, true, false, false).await +} + +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn reserved_tool_obeys_shared_concurrency_limit(db: Pool) -> anyhow::Result<()> { + run_batch(db, true, true, false).await +} + +#[sqlx::test(fixtures("base"))] +#[serial_test::serial] +async fn parent_leaves_unsupported_first_and_remaining_tools_queued( + db: Pool, +) -> anyhow::Result<()> { + run_batch(db, false, false, true).await +} diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index b7aa671b21..9616df267a 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -465,6 +465,37 @@ def main(x: int): ); } + #[test] + fn test_python_execd_async() { + let script = r#" +import asyncio + +_import_loop = asyncio.new_event_loop() +asyncio.set_event_loop(_import_loop) + +async def preprocessor(x: int): + return {"x": x * 2} + +async def main(x: int): + return {"x": x + 100, "same_loop": asyncio.get_running_loop() is _import_loop} +"#; + let results = run_py_raw_protocol_test( + &[("f/test/async", script)], + vec![ + ProtocolCmd::ExecdPreprocess { args: serde_json::json!({"x": 5}) }, + ProtocolCmd::Execd { args: serde_json::json!({"x": 7}) }, + ], + ); + assert_eq!( + results, + vec![ + DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10})), + DedicatedWorkerResult::Success(serde_json::json!({"x": 110, "same_loop": true})), + DedicatedWorkerResult::Success(serde_json::json!({"x": 107, "same_loop": true})), + ] + ); + } + // ==================== Argument Transformation Tests ==================== #[test] @@ -760,6 +791,49 @@ def main(): Ok(()) } +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base"))] +async fn test_python_async_main(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let content = r#" +import asyncio + +async def main(x: int): + await asyncio.sleep(0) + return x + 1 + "# + .to_owned(); + + let job = JobPayload::Code(RawCode { + hash: None, + content, + path: None, + language: ScriptLang::Python3, + lock: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + modules: None, + tag: None, + }); + + let result = RunJob::from(job) + .arg("x", serde_json::json!(41)) + .run_until_complete(&db, false, port) + .await + .json_result() + .unwrap(); + + assert_eq!(result, serde_json::json!(42)); + Ok(()) +} + #[cfg(feature = "python")] #[sqlx::test(fixtures("base"))] async fn test_python_result_preserves_infinity_in_string(db: Pool) -> anyhow::Result<()> { diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index 908f007978..b5b6281264 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -14,6 +14,130 @@ use std::time::{Duration, Instant}; /// its `thinking` param, Gemini to a zero budget or the model's floor). pub(crate) const REASONING_OFF_SENTINEL: &str = "none"; +/// The effort to send for a model, dropping the off sentinel on a model that rejects every +/// disable: the model then reasons at its default instead of failing the request. The UI +/// never offers off on these, so this guards an agent step saved before it stopped, or an +/// effort passed in as a flow input. +pub fn effective_reasoning_effort<'a>(model: &str, effort: Option<&'a str>) -> Option<&'a str> { + match effort { + Some(effort) + if effort == REASONING_OFF_SENTINEL + && reasoning_rule(model).is_some_and(|rule| !rule.can_disable) => + { + None + } + effort => effort, + } +} + +/// Whether a request with function tools must send the off sentinel on Chat Completions: +/// the model refuses tools there while it reasons, and does accept being turned off. +pub(crate) fn completions_tools_need_reasoning_off(model: &str) -> bool { + reasoning_rule(model).is_some_and(|rule| rule.completions_tools_need_off && rule.can_disable) +} + +/// What the backend needs to know about a model's reasoning: the rows of `REASONING_RULES` +/// in the frontend's `reasoningRegistry.ts`, cut down to the two facts the wire needs. +/// `reasoningParity.json` next to that file is checked by both sides' tests. +struct ReasoningRule { + matches: fn(&str) -> bool, + /// False when the provider rejects every disable, so the off sentinel must not be sent. + can_disable: bool, + /// Chat Completions refuses function tools while the model reasons, even with the + /// effort omitted (live-verified); the Responses API has no such limit. + completions_tools_need_off: bool, +} + +/// Matched in order against the lowercased model id, first match wins. A model no row +/// matches keeps whatever effort it was given. +const REASONING_RULES: &[ReasoningRule] = &[ + // Live-verified: Fable, Mythos and the 5.x point releases reject `thinking: disabled`. + ReasoningRule { + matches: |m| m.contains("claude-fable") || m.contains("claude-mythos"), + can_disable: false, + completions_tools_need_off: false, + }, + ReasoningRule { + matches: is_claude_5_point_release, + can_disable: false, + completions_tools_need_off: false, + }, + // Live-verified: astra takes low..max only, where sol and luna also take `none`. + ReasoningRule { + matches: |m| base_id(m).starts_with("gpt-6-astra"), + can_disable: false, + completions_tools_need_off: true, + }, + ReasoningRule { + matches: |m| gpt_version(m).is_some_and(|(major, _)| major >= 6), + can_disable: true, + completions_tools_need_off: true, + }, + ReasoningRule { + matches: |m| matches!(gpt_version(m), Some((5, Some(minor))) if minor >= 5), + can_disable: true, + completions_tools_need_off: true, + }, + ReasoningRule { + matches: |m| matches!(gpt_version(m), Some((5, Some(_)))), + can_disable: true, + completions_tools_need_off: false, + }, + // gpt-5 and the o-series reject `none`. + ReasoningRule { + matches: |m| matches!(gpt_version(m), Some((5, None))), + can_disable: false, + completions_tools_need_off: false, + }, + ReasoningRule { + matches: |m| { + let base = base_id(m); + base.starts_with('o') && base[1..].starts_with(|c: char| c.is_ascii_digit()) + }, + can_disable: false, + completions_tools_need_off: false, + }, +]; + +fn reasoning_rule(model: &str) -> Option<&'static ReasoningRule> { + let model = model.to_lowercase(); + REASONING_RULES.iter().find(|rule| (rule.matches)(&model)) +} + +/// The id after a gateway's `vendor/` and before a `:variant`. +fn base_id(model: &str) -> &str { + let last = model.rsplit('/').next().unwrap_or(model); + last.split(':').next().unwrap_or(last) +} + +/// `(major, minor)` of a `gpt-` id. The major is one digit then a separator or the end, +/// since Azure names gpt-3.5 `gpt-35-turbo`. +fn gpt_version(model: &str) -> Option<(u32, Option)> { + let rest = base_id(model).strip_prefix("gpt-")?; + let mut chars = rest.chars(); + let major = chars.next()?.to_digit(10)?; + match chars.next() { + None | Some('-') => Some((major, None)), + Some('.') => { + let minor: String = chars.take_while(char::is_ascii_digit).collect(); + Some((major, minor.parse().ok())) + } + _ => None, + } +} + +/// Sonnet or Opus 5.x with x >= 1. The version match stops at one digit so a dated id +/// (`claude-sonnet-5-20260101`) stays Sonnet 5. +fn is_claude_5_point_release(model: &str) -> bool { + let model = model.replace('.', "-"); + ["claude-opus-5-", "claude-sonnet-5-"].iter().any(|prefix| { + model.split(prefix).skip(1).any(|rest| { + let mut chars = rest.chars(); + matches!(chars.next(), Some('1'..='9')) && !matches!(chars.next(), Some('0'..='9')) + }) + }) +} + /// Whether a Claude model removed the sampling params (`temperature`, `top_p`, /// `top_k`). On these, any value is a hard 400 — `temperature is deprecated for /// this model` — whatever the thinking mode, so the param has to be dropped on @@ -119,6 +243,46 @@ pub fn remember_chat_completions_only(base_url: &str, model: &str) { CHAT_COMPLETIONS_ONLY.insert((base_url.to_string(), model.to_string()), Instant::now()); } +#[cfg(test)] +mod reasoning_rule_tests { + use super::*; + + /// The frontend registry's test reads the same file. These rules see the model id + /// alone, so it holds only rows whose answer doesn't depend on the provider. + #[test] + fn agrees_with_the_frontend_registry() { + let rows: Vec = serde_json::from_str(include_str!( + "../../../../frontend/src/lib/components/copilot/reasoningParity.json" + )) + .unwrap(); + for row in rows { + let model = row["model"].as_str().unwrap(); + let can_disable = row["canDisable"].as_bool().unwrap(); + let tools_need_off = row["completionsToolsNeedOff"].as_bool().unwrap(); + let sent = effective_reasoning_effort(model, Some(REASONING_OFF_SENTINEL)); + assert_eq!(sent.is_some(), can_disable, "{model}"); + assert_eq!( + effective_reasoning_effort(model, Some("low")), + Some("low"), + "{model}" + ); + assert_eq!( + completions_tools_need_reasoning_off(model), + tools_need_off && can_disable, + "{model}" + ); + } + } + + #[test] + fn reads_ids_the_frontend_resolves_first() { + // Azure's gpt-3.5 is not major 35, and a gateway prefix is not part of the id. + assert_eq!(gpt_version("gpt-35-turbo"), None); + assert!(completions_tools_need_reasoning_off("openai/gpt-6-sol")); + assert_eq!(effective_reasoning_effort("openai/o3", Some("none")), None); + } +} + #[cfg(test)] mod chat_completions_only_tests { use super::*; diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index b6040d8d79..1ea54a7e79 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -1,3 +1,4 @@ +use super::{completions_tools_need_reasoning_off, REASONING_OFF_SENTINEL}; use crate::{ ai_providers::AIProvider, image_handler::prepare_messages_for_api, @@ -143,6 +144,15 @@ impl OtherQueryBuilder { let (reasoning_effort, thinking, temperature) = provider_reasoning_fields(&self.provider_kind, args.reasoning_effort, args.temperature); + // With tools, gpt-5.5+ only run here with reasoning off: turn it off where the model + // can, rather than failing every turn. + let reasoning_effort = if args.tools.is_some_and(|tools| !tools.is_empty()) + && completions_tools_need_reasoning_off(args.model) + { + Some(REASONING_OFF_SENTINEL) + } else { + reasoning_effort + }; // Build request with stream_options for usage tracking let request_with_usage = OpenAICompletionRequest { diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index 03476240a3..7e2de09e01 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -294,7 +294,10 @@ impl ProviderWithResource { /// The reasoning effort to thread to the provider, treating an empty string /// (e.g. a cleared flow input) as unset. pub fn get_reasoning_effort(&self) -> Option<&str> { - self.reasoning_effort.as_deref().filter(|s| !s.is_empty()) + crate::providers::effective_reasoning_effort( + &self.model, + self.reasoning_effort.as_deref().filter(|s| !s.is_empty()), + ) } pub async fn get_base_url(&self, db: &DB) -> Result { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index d480a89851..d661483182 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -864,6 +864,24 @@ pub fn make_pull_query(tags: &[String]) -> String { query } +/// Claim a parent's tool jobs matching its worker tags. The caller must supply only child IDs +/// it owns; this query is an internal scheduling primitive and does not authorize job access. +pub fn make_tool_job_pull_query(job_ids: &[uuid::Uuid], tags: &[String]) -> String { + // pull() binds only the worker name. These literals come from typed UUIDs, never input SQL. + let ids = job_ids.iter().map(|id| format!("'{id}'::uuid")).join(", "); + let tags = tags + .iter() + .map(|tag| format!("'{}'", tag.replace('\'', "''"))) + .join(", "); + format_pull_query(format!( + "SELECT id FROM v2_job_queue + WHERE running = false AND id = ANY(ARRAY[{ids}]::uuid[]) AND scheduled_for <= now() + AND tag = ANY(ARRAY[{tags}]::text[]) + ORDER BY priority DESC NULLS LAST, scheduled_for + FOR UPDATE SKIP LOCKED LIMIT 1" + )) +} + // Variant of `make_pull_query` that additionally excludes jobs whose workspace_id is in the // overloaded-list bind parameter ($2::text[]). Built as a separate string (rather than reusing // `make_pull_query` with an always-bound array) so the planner can keep using the same indexes diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 1c72873377..448e13af85 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -184,7 +184,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28994/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -192,7 +192,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28970/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28993/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index b555e1ea92..c2c42c649c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -4909,6 +4909,45 @@ pub fn has_active_concurrency_limit(concurrent_limit: Option) -> bool { concurrent_limit.is_some_and(|n| n > 0) } +/// Admit a job already owned by a worker without releasing its queue reservation. +/// The caller must maintain its heartbeat while waiting and complete it on failure. +pub async fn try_admit_owned_job(db: &DB, job: &MiniPulledJob) -> error::Result { + #[cfg(all(feature = "private", feature = "enterprise"))] + { + let settings = windmill_common::runnable_settings::prefetch_cached_from_handle( + job.runnable_settings_handle, + db, + ) + .await? + .1 + .maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s); + if has_active_concurrency_limit(settings.concurrent_limit) + && !*DISABLE_CONCURRENCY_LIMIT + && job.canceled_by.is_none() + { + let key = concurrency_key(db, &job.id).await?.ok_or_else(|| { + Error::internal_err(format!("No concurrency key found for job {}", job.id)) + })?; + if !key.is_empty() { + return Ok(crate::jobs_ee::update_concurrency_counter( + db, + &job.id, + key, + serde_json::json!({ job.id.to_string(): {} }), + job.id.to_string(), + settings.concurrency_time_window_s.unwrap_or(0), + settings.concurrent_limit.unwrap_or_default(), + ) + .await? + .0); + } + } + } + #[cfg(not(all(feature = "private", feature = "enterprise")))] + let _ = (db, job); + Ok(true) +} + pub async fn custom_concurrency_key( db: &Pool, job_id: &Uuid, diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 682714292e..930a329be4 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -7,16 +7,15 @@ use crate::ai::utils::{ use crate::common::OccupancyMetrics; use crate::result_processor::handle_non_flow_job_error; use crate::worker_flow::{ - evaluate_input_transform, raw_script_to_payload, script_to_payload, JobPayloadWithTag, -}; -use crate::{ - create_job_dir, handle_queued_job, JobCompletedReceiver, JobCompletedSender, SendResult, - SendResultPayload, + evaluate_input_transform, raw_script_to_payload, resolve_flow_step_tag, script_to_payload, + JobPayloadWithTag, }; +use crate::{create_job_dir, handle_queued_job, JobCompletedSender}; use anyhow::Context; use mappable_rc::Marc; use serde_json::value::RawValue; -use std::{collections::HashMap, sync::Arc}; +use sqlx::types::Json; +use std::{collections::HashMap, sync::Arc, time::Duration}; use uuid::Uuid; use windmill_ai::{ai_types::OpenAIToolCall, query_builder::StreamEventSink, types::*}; use windmill_common::jobs::JobPayload; @@ -36,11 +35,12 @@ use windmill_common::{ flow_conversations::{MessageExtras, MessageType}, flow_status::AgentAction, flows::FlowModuleValue, - worker::{to_raw_value, Connection}, + worker::{make_tool_job_pull_query, to_raw_value, Connection, WORKER_CONFIG}, }; use windmill_queue::{ - add_completed_job, add_completed_job_error, check_tag_available_for_push, get_mini_pulled_job, - push, resolve_push_tag, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel, + append_logs, check_tag_available_for_push, get_mini_pulled_job, pull, push, resolve_push_tag, + tag_reads_flow_expr, try_admit_owned_job, MiniCompletedJob, MiniPulledJob, PushArgs, + PushIsolationLevel, }; /// Shared collection of abort handles for spawned tool tasks. @@ -74,6 +74,8 @@ pub struct ToolExecutionContext<'a> { pub stream_event_processor: Option<&'a StreamEventProcessor>, pub flow_context: &'a mut FlowContext, pub omit_output_from_conversation: bool, + /// The agent's flow `preserve_step_tags`: its tools pick their tag as that flow's steps do. + pub preserve_step_tags: bool, /// The thinking that led to this round's calls, stored on the first tool row written. /// None when the round wrote text, whose row carries it. pub reasoning: Option, @@ -82,6 +84,7 @@ pub struct ToolExecutionContext<'a> { // Abort handles for spawned tool tasks (used for force-cancel cleanup) pub tool_abort_handles: ToolAbortHandles, + pub job_completed_tx: JobCompletedSender, } /// Execute all tool calls from an AI response @@ -98,7 +101,8 @@ pub async fn execute_tool_calls( let mut used_structured_output_tool = false; let mut final_content = None; - for tool_call in tool_calls.iter() { + let mut calls = tool_calls.iter().peekable(); + while let Some(tool_call) = calls.next() { // Stream tool call progress if let Some(stream_event_processor) = ctx.stream_event_processor { let event = StreamingEvent::ToolExecution { @@ -150,15 +154,24 @@ pub async fn execute_tool_calls( ) .await?; } else if tool.module.is_some() { - execute_windmill_tool( - &mut ctx, - tool_call, - tool, - actions, - &mut messages, - final_events_str, - ) - .await?; + let mut batch = vec![(tool_call, tool)]; + while let Some(next_call) = calls.peek() { + if structured_output_tool_name.as_deref() + == Some(next_call.function.name.as_str()) + { + break; + } + let Some(next_tool) = tools.iter().find(|t| { + t.def.function.name == next_call.function.name + && t.mcp_source.is_none() + && t.module.is_some() + }) else { + break; + }; + batch.push((calls.next().unwrap(), next_tool)); + } + execute_windmill_tools(&mut ctx, &batch, actions, &mut messages, final_events_str) + .await?; } else { return Err(Error::internal_err(format!( "Tool type not supported: {}", @@ -313,14 +326,13 @@ async fn execute_mcp_tool_call( } /// Execute a Windmill tool (script or flow) -async fn execute_windmill_tool( - ctx: &mut ToolExecutionContext<'_>, +async fn enqueue_windmill_tool( + ctx: &ToolExecutionContext<'_>, tool_call: &OpenAIToolCall, tool: &Tool, actions: &mut Vec, - messages: &mut Vec, - final_events_str: &mut String, -) -> Result<(), Error> { + reserved: bool, +) -> Result<(Uuid, bool), Error> { // Regular Windmill tools must have a module let tool_module = tool.module.as_ref().ok_or_else(|| { Error::internal_err(format!("Tool {} has no module", tool_call.function.name)) @@ -402,8 +414,6 @@ async fn execute_windmill_tool( tool_call_args.insert(key.clone(), result); } - let is_ai_agent_tool = matches!(tool_value, FlowModuleValue::AIAgent { .. }); - let job_payload = match tool_value { FlowModuleValue::Script { path: script_path, hash: script_hash, tag_override, .. } => { script_to_payload( @@ -473,7 +483,6 @@ async fn execute_windmill_tool( )); } let path = format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id); - // a nested agent always runs inline on this worker, under this agent's tag JobPayloadWithTag { payload: JobPayload::AIAgent { path }, tag: None, @@ -502,18 +511,34 @@ async fn execute_windmill_tool( let push_args = PushArgs { args: &tool_call_args, extra: None }; - // A tool with a tag of its own naming another queue than this agent's must run on a worker - // that tag selects, and like any tag the caller chose it must pass the workspace's custom tag - // restrictions. It runs inline when this worker serves that queue: the agent holds this - // worker while it waits, so dispatching to a queue it serves could wait on itself. - let mut resolved_tool_tag = None; - if let Some(tag) = job_payload.tag.as_deref() { - resolved_tool_tag = resolve_push_tag(tag, &push_args, &ctx.job.workspace_id, ctx.db) - .await - .filter(|resolved| *resolved != ctx.job.tag); + // The agent's tag stands in for the flow's, as a sub-flow's does for its own steps. + let tool_tag = resolve_flow_step_tag( + false, + &ctx.job.tag, + &ctx.job.workspace_id, + ctx.preserve_step_tags, + job_payload.tag.as_deref(), + ); + if tool_tag.as_deref().is_some_and(tag_reads_flow_expr) { + append_logs( + &ctx.job.id, + &ctx.job.workspace_id, + format!( + "Tool '{}' is tagged with `$flow_expr[...]`, which AI agent tools do not resolve, so it runs on its default tag.\n", + tool_call.function.name + ), + ctx.conn, + ) + .await; } - let tool_tag = match (job_payload.tag.as_deref(), resolved_tool_tag.as_ref()) { - (Some(tag), Some(_)) => { + + // A tool's own tag routes its job, so like any tag the caller chose it must pass the + // workspace's custom tag restrictions. The agent's tag was already checked when it was pushed. + if let Some(tag) = tool_tag.as_deref() { + if resolve_push_tag(tag, &push_args, &ctx.job.workspace_id, ctx.db) + .await + .is_some_and(|resolved| resolved != ctx.job.tag) + { let is_super_admin = windmill_common::auth::is_super_admin_email(ctx.db, email).await?; check_tag_available_for_push( ctx.db, @@ -531,16 +556,8 @@ async fn execute_windmill_tool( )), e => e, })?; - Some(tag.to_string()) } - _ => None, - }; - let run_inline = resolved_tool_tag.is_none_or(|resolved| { - windmill_common::worker::WORKER_CONFIG - .load() - .worker_tags - .contains(&resolved) - }); + } let mut tx = ctx.db.begin().await?; @@ -573,77 +590,71 @@ async fn execute_windmill_tool( false, None, ctx.job.visible_to_owner, - Some(tool_tag.unwrap_or_else(|| ctx.job.tag.clone())), + tool_tag, job_payload.timeout, None, job_priority, job_perms.as_ref(), - run_inline, + false, None, None, None, ) .await?; - tx.commit().await?; - - if !run_inline { - // Like an inline tool's failure, a lost wait reaches the model rather than failing the - // agent. - let (result, success) = - match wait_for_dispatched_tool_job(ctx.db, &uuid, &ctx.job.workspace_id).await { - Ok(outcome) => outcome, - Err(e) => ( - to_raw_value(&serde_json::json!({ "error": { "message": e.to_string() } })), - false, - ), - }; - return report_tool_result( - ctx, - tool_call, - tool_module, - job_id, - &result, - success, - is_ai_agent_tool && success, - messages, - final_events_str, + let mut tx = tx; + let reserved = if reserved { + let tags = local_tool_tags(); + // Reserve a supported first tool before commit so other workers cannot race it. + let claimed = sqlx::query!( + "UPDATE v2_job_queue SET worker = $1, running = true, started_at = now() + WHERE id = $2 AND tag = ANY($3)", + ctx.worker_name, + uuid, + &tags, ) - .await; - } - - let tool_job = get_mini_pulled_job(ctx.db, &uuid).await?; - - let Some(tool_job) = tool_job else { - return Err(Error::internal_err("Tool job not found".to_string())); + .execute(&mut *tx) + .await? + .rows_affected() + == 1; + if claimed { + sqlx::query!("UPDATE v2_job_runtime SET ping = now() WHERE id = $1", uuid) + .execute(&mut *tx) + .await?; + } + claimed + } else { + false }; + tx.commit().await?; + Ok((uuid, reserved)) +} - // The tool gets a token of its own, as it would on any worker: the agent's token would - // identify the tool as the agent job (its OIDC path and job id). - let tool_client = AuthedClient::new( - ctx.base_internal_url.to_string(), - tool_job.workspace_id.clone(), - windmill_queue::create_token(ctx.db, &tool_job, None).await, - None, - ); - let tool_job = Arc::new(tool_job); +fn local_tool_tags() -> Vec { + WORKER_CONFIG + .load() + .priority_tags_sorted + .iter() + .flat_map(|group| group.tags.iter().cloned()) + .collect() +} - let (inner_job_completed_tx, inner_job_completed_rx) = JobCompletedSender::new(ctx.conn, 1); - - let inner_job_completed_rx = inner_job_completed_rx.expect( - "inner_job_completed_tx should be set as agent jobs are not supported on agent workers", - ); +fn spawn_local_tool( + ctx: &ToolExecutionContext<'_>, + tool_job: MiniPulledJob, + reserved: bool, +) -> tokio::task::JoinHandle> { + let mut tool_job = Arc::new(tool_job); // Spawn handle_queued_job on separate task to prevent tokio stack overflow // Clone everything needed for the spawned task - let tool_job_spawn = tool_job.clone(); + let db = ctx.db.clone(); let conn_spawn = ctx.conn.clone(); - let client_spawn = tool_client; let hostname_spawn = ctx.hostname.to_string(); let worker_name_spawn = ctx.worker_name.to_string(); let worker_dir_spawn = ctx.worker_dir.to_string(); let base_internal_url_spawn = ctx.base_internal_url.to_string(); - let inner_job_completed_tx_spawn = inner_job_completed_tx.clone(); + let job_completed_tx = ctx.job_completed_tx.clone(); let mut occupancy_metrics_spawn = ctx.occupancy_metrics.clone(); let mut killpill_rx_spawn = ctx.killpill_rx.resubscribe(); @@ -652,147 +663,270 @@ async fn execute_windmill_tool( #[cfg(feature = "benchmark")] let mut bench_spawn = windmill_common::bench::BenchmarkIter::new(); - let job_dir = create_job_dir(&worker_dir_spawn, tool_job_spawn.id).await; + let result = async { + if reserved { + loop { + let queued = get_mini_pulled_job(&db, &tool_job.id) + .await? + .ok_or_else(|| Error::AlreadyCompleted("Tool job already completed".to_string()))?; + tool_job = Arc::new(queued); + sqlx::query!( + "UPDATE v2_job_runtime SET ping = now() WHERE id = $1", + tool_job.id, + ) + .execute(&db) + .await?; + if try_admit_owned_job(&db, &tool_job).await? { + let started_at = sqlx::query_scalar!( + "UPDATE v2_job_queue SET started_at = now() WHERE id = $1 RETURNING started_at", + tool_job.id, + ) + .fetch_optional(&db) + .await? + .flatten(); + Arc::make_mut(&mut tool_job).started_at = started_at; + break; + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + } + let perms = + windmill_common::auth::get_job_perms(&db, &tool_job.id, &tool_job.workspace_id).await?; + let token = windmill_queue::create_token(&db, &tool_job, perms).await; + let client_spawn = AuthedClient::new( + base_internal_url_spawn.clone(), + tool_job.workspace_id.clone(), + token, + None, + ); + let job_dir = create_job_dir(&worker_dir_spawn, tool_job.id).await; - let result = handle_queued_job( - tool_job_spawn, - None, - None, - None, - None, - &conn_spawn, - &client_spawn, - &hostname_spawn, - &worker_name_spawn, - &worker_dir_spawn, - &job_dir, - None, - &base_internal_url_spawn, - inner_job_completed_tx_spawn, - &mut occupancy_metrics_spawn, - &mut killpill_rx_spawn, - None, - None, - #[cfg(feature = "benchmark")] - &mut bench_spawn, - ) + handle_queued_job( + tool_job.clone(), + None, + None, + None, + None, + &conn_spawn, + &client_spawn, + &hostname_spawn, + &worker_name_spawn, + &worker_dir_spawn, + &job_dir, + None, + &base_internal_url_spawn, + job_completed_tx, + &mut occupancy_metrics_spawn, + &mut killpill_rx_spawn, + None, + None, + #[cfg(feature = "benchmark")] + &mut bench_spawn, + ) + .await + } .await; - // Return both result and updated metrics - (result, occupancy_metrics_spawn) + match result { + Err(err) => { + let err_string = format!("{}: {}", err.name(), err); + handle_non_flow_job_error( + &db, + &MiniCompletedJob::from(tool_job), + 0, + None, + err_string, + windmill_common::worker::error_to_value(&err), + &worker_name_spawn, + ) + .await?; + } + Ok(_) => {} + } + Ok(occupancy_metrics_spawn) }); // Register abort handle so the task can be killed on force-cancel let abort_handle = join_handle.abort_handle(); // unwrap safe: lock is only held briefly for push/drain, no panic possible inside ctx.tool_abort_handles.lock().unwrap().push(abort_handle); - - // Await the spawned task - let (handle_result, updated_occupancy) = join_handle.await.map_err(|e| { - if e.is_cancelled() { - Error::ExecutionErr("Tool execution task was cancelled".to_string()) - } else { - Error::internal_err(format!("Tool execution task failed: {}", e)) - } - })?; - - // Merge occupancy metrics back - ctx.occupancy_metrics.total_duration_of_running_jobs = - updated_occupancy.total_duration_of_running_jobs; - - match handle_result { - Err(err) => { - handle_tool_execution_error( - ctx, - tool_call, - tool_module, - &MiniCompletedJob::from(tool_job), - job_id, - err, - messages, - final_events_str, - ) - .await?; - } - Ok(outcome) => { - handle_tool_execution_success( - ctx, - tool_call, - tool_module, - job_id, - outcome.is_success(), - is_ai_agent_tool, - inner_job_completed_rx, - messages, - final_events_str, - ) - .await?; - } - } - - Ok(()) + join_handle } -/// Handle tool execution error -async fn handle_tool_execution_error( +async fn execute_windmill_tools( ctx: &mut ToolExecutionContext<'_>, - tool_call: &OpenAIToolCall, - tool_module: &windmill_common::flows::FlowModule, - tool_job: &MiniCompletedJob, - job_id: Uuid, - err: Error, + batch: &[(&OpenAIToolCall, &Tool)], + actions: &mut Vec, messages: &mut Vec, final_events_str: &mut String, ) -> Result<(), Error> { - let err_string = format!("{}: {}", err.name(), err.to_string()); - let err_json = windmill_common::worker::error_to_value(&err); - let _ = handle_non_flow_job_error( - ctx.db, - tool_job, - 0, - None, - err_string.clone(), - err_json, - ctx.worker_name, - ) - .await; - - let error_message = format!("Error running tool: {}", err_string); - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(error_message.clone())), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool_module.id.clone(), - }), - ..Default::default() - }); - - // Stream tool result (error case) - if let Some(stream_event_processor) = ctx.stream_event_processor { - let tool_result_event = StreamingEvent::ToolResult { - call_id: tool_call.id.clone(), - function_name: tool_call.function.name.clone(), - result: error_message.clone(), - success: false, - }; - stream_event_processor - .send(tool_result_event, final_events_str) - .await?; + let mut job_ids = Vec::with_capacity(batch.len()); + let mut local = None; + for (index, (call, tool)) in batch.iter().enumerate() { + if index > 0 { + if let Some(processor) = ctx.stream_event_processor { + processor + .send( + StreamingEvent::ToolExecution { + call_id: call.id.clone(), + function_name: call.function.name.clone(), + }, + final_events_str, + ) + .await?; + } + } + let (id, reserved) = enqueue_windmill_tool(ctx, call, tool, actions, index == 0).await?; + job_ids.push(id); + if reserved { + let job = get_mini_pulled_job(ctx.db, &id) + .await? + .ok_or_else(|| Error::internal_err("Reserved tool job not found".to_string()))?; + local = Some(spawn_local_tool(ctx, job, true)); + } } - if let Some(parent_job) = ctx.parent_job { - update_flow_status_module_with_actions_success(ctx.db, parent_job, false).await?; + let mut results: HashMap = HashMap::new(); + let mut next_result = 0; + let mut poll = tokio::time::interval(Duration::from_millis(100)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + while next_result < batch.len() || local.is_some() { + tokio::select! { + result = async { local.as_mut().unwrap().await }, if local.is_some() => { + local = None; + let metrics = result.map_err(|e| Error::internal_err(format!("Tool task failed: {e}")))??; + ctx.occupancy_metrics.total_duration_of_running_jobs = metrics.total_duration_of_running_jobs; + ctx.tool_abort_handles.lock().unwrap().retain(|handle| !handle.is_finished()); + } + _ = poll.tick() => {} + } + + let pending: Vec = job_ids[next_result..] + .iter() + .filter(|id| !results.contains_key(id)) + .copied() + .collect(); + if !pending.is_empty() { + let completed = sqlx::query!( + "SELECT id, status = 'success' AS \"success!\", result AS \"result: Json>\" + FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)", + ctx.job.workspace_id, + &pending, + ).fetch_all(ctx.db).await?; + for completed in completed { + let index = job_ids + .iter() + .position(|id| *id == completed.id) + .ok_or_else(|| Error::internal_err("Unexpected tool completion".to_string()))?; + let (call, tool) = batch[index]; + let result = completed + .result + .map(|value| value.0) + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let is_agent = tool.module.as_ref().is_some_and(|module| { + matches!(module.get_value(), Ok(FlowModuleValue::AIAgent { .. })) + }); + let content = if is_agent && completed.success { + extract_ai_agent_output(&result).unwrap_or_else(|| result.get().to_string()) + } else { + result.get().to_string() + }; + if let Some(processor) = ctx.stream_event_processor { + processor + .send( + StreamingEvent::ToolResult { + call_id: call.id.clone(), + function_name: call.function.name.clone(), + result: content.clone(), + success: completed.success, + }, + final_events_str, + ) + .await?; + } + results.insert(completed.id, (completed.success, content)); + } + } + + // Transcript rows, model messages, and positional action statuses share call order. + while next_result < batch.len() { + let job_id = job_ids[next_result]; + let Some((success, content)) = results.remove(&job_id) else { + break; + }; + let (call, tool) = batch[next_result]; + let module = tool + .module + .as_ref() + .ok_or_else(|| Error::internal_err("Windmill tool has no module".to_string()))?; + messages.push(OpenAIMessage { + role: "tool".to_string(), + content: Some(OpenAIContent::Text(content.clone())), + tool_call_id: Some(call.id.clone()), + agent_action: Some(AgentAction::ToolCall { + job_id, + function_name: call.function.name.clone(), + module_id: module.id.clone(), + }), + ..Default::default() + }); + if let Some(parent) = ctx.parent_job { + update_flow_status_module_with_actions_success(ctx.db, parent, success).await?; + } + let (content, extras) = windmill_tool_row(call, success, &content); + add_tool_message_to_chat(ctx, Some(job_id), &content, success, Some(extras)).await; + next_result += 1; + } + + if local.is_none() && next_result < batch.len() { + let pending: Vec = job_ids[next_result..] + .iter() + .filter(|id| !results.contains_key(id)) + .copied() + .collect(); + if !pending.is_empty() { + local = claim_local_tool(ctx, &pending).await?; + } + } } - - let (content, extras) = windmill_tool_row(tool_call, false, &error_message); - add_tool_message_to_chat(ctx, Some(job_id), &content, false, Some(extras)).await; - Ok(()) } +async fn claim_local_tool( + ctx: &ToolExecutionContext<'_>, + pending: &[Uuid], +) -> Result>>, Error> { + let query = ( + String::new(), + make_tool_job_pull_query(pending, &local_tool_tags()), + ); + #[cfg(feature = "benchmark")] + let mut bench = windmill_common::bench::BenchmarkIter::new(); + let mut pulled = pull( + ctx.db, + false, + ctx.worker_name, + Some(&query), + #[cfg(feature = "benchmark")] + &mut bench, + ) + .await?; + if let Err(err) = pulled.maybe_apply_debouncing(ctx.db).await { + pulled.error_while_preprocessing = Some(err.to_string()); + } + match pulled.to_pulled_job() { + Ok(job) => Ok(job.map(|job| spawn_local_tool(ctx, job.job, false))), + Err( + windmill_queue::PulledJobResultToJobErr::MissingConcurrencyKey(job) + | windmill_queue::PulledJobResultToJobErr::ErrorWhilePreprocessing(job), + ) => { + ctx.job_completed_tx.send_job(job, true).await?; + Ok(None) + } + } +} + /// Extract the `output` field of an `AIAgentResult` envelope, serialized back to JSON. /// Returns `None` if the result is not a JSON object carrying an `output` field. fn extract_ai_agent_output(result: &RawValue) -> Option { @@ -802,200 +936,6 @@ fn extract_ai_agent_output(result: &RawValue) -> Option { .map(|output| output.get().to_string()) } -/// Handle tool execution success -async fn handle_tool_execution_success( - ctx: &mut ToolExecutionContext<'_>, - tool_call: &OpenAIToolCall, - tool_module: &windmill_common::flows::FlowModule, - job_id: Uuid, - success: bool, - is_ai_agent_tool: bool, - inner_job_completed_rx: JobCompletedReceiver, - messages: &mut Vec, - final_events_str: &mut String, -) -> Result<(), Error> { - let send_result = inner_job_completed_rx.bounded_rx.try_recv().ok(); - - let (result, job_success) = if let Some(SendResult { - result: SendResultPayload::JobCompleted(ref jc), - .. - }) = send_result - { - let result = jc.result.clone(); - // Write tool completion to the DB inline instead of forwarding through - // the parent channel. Forwarding would deadlock for nested agents: the - // sub-tool result would fill the parent's bounded(1) channel, leaving - // no room for the agent's own completion from process_result. - if jc.success { - add_completed_job( - ctx.db, - &jc.job, - true, - false, - sqlx::types::Json(&*jc.result), - jc.result_columns.clone(), - jc.mem_peak, - jc.canceled_by.clone(), - false, - jc.duration, - jc.from_cache.unwrap_or(false), - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to add completed job: {e}")))?; - } else { - let error_value: serde_json::Value = - serde_json::from_str(jc.result.get()).unwrap_or_else(|_| { - serde_json::json!({ "message": format!("Non serializable error: {}", jc.result.get()) }) - }); - add_completed_job_error( - ctx.db, - &jc.job, - jc.mem_peak, - jc.canceled_by.clone(), - error_value, - ctx.worker_name, - false, - jc.duration, - ) - .await - .map_err(|e| Error::internal_err(format!("Failed to add completed job error: {e}")))?; - } - (result, jc.success) - } else { - return Err(Error::internal_err( - "Tool job completed but no result".to_string(), - )); - }; - - report_tool_result( - ctx, - tool_call, - tool_module, - job_id, - &result, - success, - is_ai_agent_tool && job_success, - messages, - final_events_str, - ) - .await -} - -/// Wait for a tool job another worker runs; returns its result and whether it succeeded. -/// Canceling the agent cancels this job along with it, as a child of the agent's job. -async fn wait_for_dispatched_tool_job( - db: &DB, - id: &Uuid, - w_id: &str, -) -> Result<(Box, bool), Error> { - const MAX_CONSECUTIVE_POLL_ERRORS: u32 = 10; - let mut interval = std::time::Duration::from_millis(50); - let mut poll_errors = 0; - loop { - // One statement reads both tables from one snapshot, so a job completing between two - // reads cannot look like one that vanished. - let state = match sqlx::query!( - "SELECT c.id IS NOT NULL AS \"completed!\", - c.result AS \"result: sqlx::types::Json>\", - c.status = 'success' AS \"success\", - EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1) AS \"queued!\" - FROM (SELECT 1) one - LEFT JOIN v2_job_completed c ON c.id = $1 AND c.workspace_id = $2", - id, - w_id - ) - .fetch_one(db) - .await - { - Ok(state) => { - poll_errors = 0; - state - } - Err(e) if poll_errors < MAX_CONSECUTIVE_POLL_ERRORS => { - poll_errors += 1; - tracing::warn!("polling tool job {id} failed, retrying: {e}"); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - continue; - } - Err(e) => return Err(e.into()), - }; - if state.completed { - let result = state - .result - .map(|r| r.0) - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); - return Ok((result, state.success.unwrap_or(false))); - } - if !state.queued { - return Err(Error::internal_err(format!( - "tool job {id} is neither queued nor completed" - ))); - } - tokio::time::sleep(interval).await; - interval = std::cmp::min(interval * 2, std::time::Duration::from_secs(1)); - } -} - -/// Hand a finished tool job's result to the model, the stream, the flow status and the chat. -async fn report_tool_result( - ctx: &mut ToolExecutionContext<'_>, - tool_call: &OpenAIToolCall, - tool_module: &windmill_common::flows::FlowModule, - job_id: Uuid, - result: &RawValue, - success: bool, - is_ai_agent_output: bool, - messages: &mut Vec, - final_events_str: &mut String, -) -> Result<(), Error> { - // A nested agent returns the whole `AIAgentResult` envelope: on top of `output` it carries - // the child's entire message history, stream log and token usage. Feeding that back would - // grow the caller's context by the child's full transcript on every call, so the caller only - // sees `output`. The envelope stays intact in the tool job's completed row. - let tool_result = if is_ai_agent_output { - extract_ai_agent_output(result).unwrap_or_else(|| result.get().to_string()) - } else { - result.get().to_string() - }; - - messages.push(OpenAIMessage { - role: "tool".to_string(), - content: Some(OpenAIContent::Text(tool_result.clone())), - tool_call_id: Some(tool_call.id.clone()), - agent_action: Some(AgentAction::ToolCall { - job_id, - function_name: tool_call.function.name.clone(), - module_id: tool_module.id.clone(), - }), - ..Default::default() - }); - - let (content, extras) = windmill_tool_row(tool_call, success, &tool_result); - - // The job ran; whether it ran successfully is `success`, and the row stored below is - // worded from it. The stream has to carry the same value, or the card the reader watches - // and the row that replaces it describe the same call differently. - if let Some(stream_event_processor) = ctx.stream_event_processor { - let tool_result_event = StreamingEvent::ToolResult { - call_id: tool_call.id.clone(), - function_name: tool_call.function.name.clone(), - result: tool_result, - success, - }; - stream_event_processor - .send(tool_result_event, final_events_str) - .await?; - } - - if let Some(parent_job) = ctx.parent_job { - update_flow_status_module_with_actions_success(ctx.db, parent_job, success).await?; - } - - add_tool_message_to_chat(ctx, Some(job_id), &content, success, Some(extras)).await; - - Ok(()) -} - /// A Windmill tool's conversation row: worded from the tool, carrying the model's call and /// the exact text the model got back, the same text agent memory keeps for that tool /// message, so a card needs no job fetch. The call is the model's arguments, not the job's @@ -1060,9 +1000,7 @@ async fn add_tool_message_to_chat( .or(ctx.job.flow_step_id.as_deref()); let step_name = get_step_name_from_flow(ctx.summary.as_deref(), effective_step_id); - // Awaited, not spawned: `created_seq` is the transcript's order, so a round's rows - // must commit in the order of its calls. Calls run one after another; running them - // in parallel would need their rows written in call order all the same. + // created_seq defines transcript order, so these writes must stay sequential. if let Err(e) = add_message_to_conversation( ctx.db, &memory_id, diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index d7ded3652f..958caeaefc 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -607,6 +607,7 @@ pub async fn handle_ai_agent_job( hostname: &str, killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, has_stream: &mut bool, + job_completed_tx: crate::JobCompletedSender, ) -> Result, Error> { // build_args_map returns None if no $res:/$var: transforms needed, in which case use original args let local_args = match build_args_map(job, client, conn).await? { @@ -686,6 +687,7 @@ pub async fn handle_ai_agent_job( }; let value = flow_data.value(); + let preserve_step_tags = value.preserve_step_tags; let module = if direct_parent_job_kind == JobKind::AIAgent { let parent_agent_step_id = direct_parent_job_flow_step_id.as_deref().ok_or_else(|| { @@ -1111,8 +1113,10 @@ pub async fn handle_ai_agent_job( has_stream, has_websearch, omit_output_from_conversation, + preserve_step_tags, cancel_rx, tool_abort_handles.clone(), + job_completed_tx, ); let mut occupancy_opt = Some(occupancy_metrics); @@ -1131,13 +1135,10 @@ pub async fn handle_ai_agent_job( cancel_tx, CANCEL_GRACE_PERIOD, ) - .await? + .await }; // agent_fut and update_job are now dropped — borrows on mcp_clients and canceled_by released - // Cleanup MCP clients - cleanup_mcp_clients(mcp_clients).await; - let format_cancel_info = |cb: &Option| { cb.as_ref() .map_or(("unknown".to_string(), "unknown".to_string()), |x| { @@ -1148,43 +1149,42 @@ pub async fn handle_ai_agent_job( }) }; - match outcome { - GracefulPollOutcome::Ok(result) => Ok(result), - GracefulPollOutcome::Timeout(ms) => { - tracing::error!("AI agent timeout after {}s", ms / 1000); - // A tool dispatched to another worker group would otherwise still run later, after - // the agent that wanted its result is gone. - let reason = format!("parent AI agent {} timed out", job.id); - let cb = CanceledBy { username: None, reason: Some(reason) }; - cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, Some(cb)).await; - Err(Error::ExecutionErr(format!( - "AI agent timeout after (>{}s)", - ms / 1000 - ))) - } - GracefulPollOutcome::Cancelled { canceled_by: cb } => { - let (by, reason) = format_cancel_info(&cb); - Err(Error::ExecutionErr(format!( - "Job cancelled by {by} (reason: {reason})" - ))) - } - GracefulPollOutcome::CancelledTimeout { canceled_by: cb } => { - let (by, reason) = format_cancel_info(&cb); - // Abort any still-running spawned tool tasks - // unwrap safe: lock is only held briefly for push/drain, no panic possible inside - for handle in tool_abort_handles.lock().unwrap().drain(..) { - handle.abort(); + let result = match outcome { + Err(error) => Err(error), + Ok(outcome) => match outcome { + GracefulPollOutcome::Ok(result) => Ok(result), + GracefulPollOutcome::Timeout(ms) => { + tracing::error!("AI agent timeout after {}s", ms / 1000); + Err(Error::ExecutionErr(format!( + "AI agent timeout after (>{}s)", + ms / 1000 + ))) } - // Hard timeout: clean up orphaned jobs still stuck in v2_job_queue - cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, cb).await; - Err(Error::ExecutionErr(format!( - "Job cancelled by {by} (reason: {reason}, timed out waiting for tool calls)" - ))) - } - GracefulPollOutcome::AlreadyCompleted => { - Err(Error::AlreadyCompleted("Job already completed".to_string())) + GracefulPollOutcome::Cancelled { canceled_by: cb } => { + let (by, reason) = format_cancel_info(&cb); + Err(Error::ExecutionErr(format!( + "Job cancelled by {by} (reason: {reason})" + ))) + } + GracefulPollOutcome::CancelledTimeout { canceled_by: cb } => { + let (by, reason) = format_cancel_info(&cb); + Err(Error::ExecutionErr(format!( + "Job cancelled by {by} (reason: {reason}, timed out waiting for tool calls)" + ))) + } + GracefulPollOutcome::AlreadyCompleted => { + Err(Error::AlreadyCompleted("Job already completed".to_string())) + } + }, + }; + if result.is_err() { + for handle in tool_abort_handles.lock().unwrap().drain(..) { + handle.abort(); } + cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, canceled_by.clone()).await; } + cleanup_mcp_clients(mcp_clients).await; + result } /// OpenAI rejects a `prompt_cache_key` over 64 characters @@ -1231,12 +1231,14 @@ pub async fn run_agent( has_stream: &mut bool, has_websearch: bool, omit_output_from_conversation: bool, + preserve_step_tags: bool, // cancellation signal from parent cancel_rx: tokio::sync::watch::Receiver, // abort handles for spawned tool tasks tool_abort_handles: ToolAbortHandles, + job_completed_tx: crate::JobCompletedSender, ) -> error::Result> { let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text); let credentials = args.provider.to_provider_credentials(db).await?; @@ -2105,6 +2107,7 @@ pub async fn run_agent( stream_event_processor: stream_event_processor.as_ref(), flow_context: &mut flow_context, omit_output_from_conversation, + preserve_step_tags, reasoning: if structured_output_first { None } else { @@ -2113,6 +2116,7 @@ pub async fn run_agent( previous_result: &previous_result, id_context: &id_context, tool_abort_handles: tool_abort_handles.clone(), + job_completed_tx: job_completed_tx.clone(), }; let (tool_messages, tool_content, tool_used_structured_output) = @@ -3093,11 +3097,13 @@ async fn cleanup_orphaned_tool_jobs( ) }); - // Find direct child jobs still in v2_job_queue (agent tool jobs are always direct children) let orphaned_ids: Vec = match sqlx::query_scalar!( - r#"SELECT j.id FROM v2_job j - JOIN v2_job_queue q ON q.id = j.id - WHERE j.parent_job = $1 AND j.workspace_id = $2"#, + r#"WITH RECURSIVE descendants AS ( + SELECT id FROM v2_job WHERE parent_job = $1 AND workspace_id = $2 + UNION ALL + SELECT j.id FROM v2_job j JOIN descendants d ON j.parent_job = d.id + WHERE j.workspace_id = $2 + ) SELECT d.id AS "id!" FROM descendants d JOIN v2_job_queue q ON q.id = d.id"#, parent_job_id, w_id, ) diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index d3bd976f21..b8c850a2f4 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2969,7 +2969,7 @@ mod tests { assert!( attach.starts_with(&format!( "ATTACH 'sslmode=verify-full sslrootcert=''{}''", - root + root.replace('\\', "\\\\") )), "{attach}" ); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 5f5ca170b4..6b24b93e77 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -828,7 +828,7 @@ pub async fn handle_python_job( for k, v in list(pre_args.items()): if v == '': del pre_args[k] - kwargs = inner_script.preprocessor(**pre_args) + kwargs = _await(inner_script.preprocessor(**pre_args)) kwrags_json = res_to_json(kwargs, type(kwargs)) with open("args.json", 'w', encoding="utf-8") as f: f.write(kwrags_json)"# @@ -969,6 +969,8 @@ result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.j def res_to_json(res, typ): {res_to_json_body} +{PY_AWAIT_HELPER} + try: {preprocessor} {spread} @@ -977,7 +979,7 @@ try: del args[k] if inner_script.{main_override} is None or not callable(inner_script.{main_override}): raise ValueError("{main_override} function is missing") - res = inner_script.{main_override}(**args) + res = _await(inner_script.{main_override}(**args)) typ = type(res) if hasattr(res, '__iter__') and not isinstance(res, (str, dict, list, bytes, tuple, set, frozenset, range, memoryview, bytearray)) and typ.__name__ != 'DataFrame': for chunk in res: @@ -1513,6 +1515,8 @@ _fix=lambda s:s if 'Infinity' not in s and 'NaN' not in s and '\\u0000' not in s def res_to_json(res, typ): {res_to_json_body} + +{PY_AWAIT_HELPER} {functions} {registrations} @@ -1546,11 +1550,11 @@ for line in sys.stdin: continue kwargs = json.loads(args_json, strict=False) pre_args = entry['pre_transform'](kwargs) - preprocessed = mod.preprocessor(**pre_args) + preprocessed = _await(mod.preprocessor(**pre_args)) preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '') sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n") main_args = entry['transform'](preprocessed if preprocessed else {{}}) - res = mod.main(**main_args) + res = _await(mod.main(**main_args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -1574,11 +1578,11 @@ for line in sys.stdin: continue kwargs = json.loads(args_json, strict=False) pre_args = entry['pre_transform'](kwargs) - preprocessed = mod.preprocessor(**pre_args) + preprocessed = _await(mod.preprocessor(**pre_args)) preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '') sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n") main_args = entry['transform'](preprocessed if preprocessed else {{}}) - res = mod.main(**main_args) + res = _await(mod.main(**main_args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -1596,7 +1600,7 @@ for line in sys.stdin: entry = next(iter(scripts.values())) kwargs = json.loads(args_json, strict=False) args = entry['transform'](kwargs) - res = entry['mod'].main(**args) + res = _await(entry['mod'].main(**args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -1624,7 +1628,7 @@ for line in sys.stdin: kwargs = json.loads(args_json, strict=False) args = entry['transform'](kwargs) - res = entry['mod'].main(**args) + res = _await(entry['mod'].main(**args)) typ = type(res) res_json = res_to_json(res, typ) sys.stdout.write("wm_res[success]:" + res_json + "\n") @@ -3330,6 +3334,31 @@ This is not normal behavior, please make sure all workers have enough memory.\n }; } +/// Python helper running what an `async def` main or preprocessor returns. +/// One loop per process: a dedicated worker's module-level async clients and +/// locks are bound to the loop they were first used on, so `asyncio.run`'s +/// loop-per-call would break them from the second job on. A loop the script +/// set while importing (`asyncio.set_event_loop`) is reused. asyncio is +/// imported lazily so sync jobs don't pay its import time. +const PY_AWAIT_HELPER: &str = r#"_loop = None +def _await(r): + global _loop + if not hasattr(r, '__await__'): + return r + import asyncio + if _loop is None or _loop.is_closed(): + import warnings + try: + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + _loop = asyncio.get_event_loop() + if _loop.is_closed(): + raise RuntimeError + except RuntimeError: + _loop = asyncio.new_event_loop() + asyncio.set_event_loop(_loop) + return _loop.run_until_complete(r)"#; + /// Python function body for `res_to_json(res, typ)`. /// Handles DataFrame, bytes, dict coercion + JSON serialization. fn python_res_to_json_body(postprocessor: &str) -> String { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index f45c295b7a..ce3fe26545 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4990,6 +4990,7 @@ pub async fn handle_queued_job( hostname, killpill_rx, &mut has_stream, + job_completed_tx.clone(), )) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index a3257ace36..7c1ea23de4 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -3239,7 +3239,7 @@ struct PushNextFlowJobRec { /// - when the flow opts into `preserve_step_tags` and the child declares its own non-empty tag, /// that tag is honored instead of being overridden by the flow tag; /// - otherwise the child inherits the parent flow job's tag. -fn resolve_flow_step_tag( +pub(crate) fn resolve_flow_step_tag( is_preprocessor_step: bool, flow_tag: &str, workspace_id: &str, diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 51f8b17818..9dded869e1 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -77,6 +77,7 @@ import { import { getEffectiveSettings, inferWsNameFromProfile, + inferWsNameFromForkBranch, resolveWsNameForConfigFromFlags, mergeConfigWithConfigFile, parseSyncBehavior, @@ -3752,9 +3753,12 @@ export async function pull( } } - // If wsNameForConfig wasn't set from flags, infer from the resolved profile + // If wsNameForConfig wasn't set from flags, infer from the resolved profile, + // then (for a fork) from the parent's entry if (!wsNameForConfig) { - wsNameForConfig = inferWsNameFromProfile(opts, workspace); + wsNameForConfig = + inferWsNameFromProfile(opts, workspace) ?? + inferWsNameFromForkBranch(opts, workspace.workspaceId); } // Resolve effective sync options with workspace awareness @@ -4802,9 +4806,12 @@ export async function push( const workspace = await resolveWorkspace(opts, wsNameForConfig); await requireLogin(opts); - // If wsNameForConfig wasn't set from flags, infer from the resolved profile + // If wsNameForConfig wasn't set from flags, infer from the resolved profile, + // then (for a fork) from the parent's entry if (!wsNameForConfig) { - wsNameForConfig = inferWsNameFromProfile(opts, workspace); + wsNameForConfig = + inferWsNameFromProfile(opts, workspace) ?? + inferWsNameFromForkBranch(opts, workspace.workspaceId); } // Resolve effective sync options with workspace awareness diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 7c6eb13274..455e876292 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -5,6 +5,7 @@ import { stringify as yamlStringify } from "yaml"; import { getCurrentGitBranch, getOriginalBranchForWorkspaceForks, + getWorkspaceIdForWorkspaceForkFromBranchName, isGitRepository, } from "../utils/git.ts"; import { join, dirname, resolve, relative } from "node:path"; @@ -725,6 +726,30 @@ export function inferWsNameFromProfile( return undefined; } +/** + * The workspace config entry a fork inherits from its parent: the one bound to + * the base branch of the checked-out `wm-fork//` branch, when + * `workspaceId` is the fork that branch names. A fork id never has an entry of + * its own, so without this its specific items and file suffix fall back to + * the common/base files and clobber what the fork cloned from its parent. + */ +export function inferWsNameFromForkBranch( + opts: SyncOptions, + workspaceId: string +): string | undefined { + if (!opts.workspaces || !isGitRepository()) return undefined; + const branch = getCurrentGitBranch(); + const baseBranch = getOriginalBranchForWorkspaceForks(branch); + if ( + !branch || + !baseBranch || + getWorkspaceIdForWorkspaceForkFromBranchName(branch) !== workspaceId + ) { + return undefined; + } + return findWorkspaceByGitBranch(opts.workspaces, baseBranch)?.[0]; +} + /** * `syncBehavior` as the workspace being pushed to sees it. The top level alone * misses a `workspaces..overrides.syncBehavior`, which is where a repo diff --git a/cli/test/fork_specific_items_unit.test.ts b/cli/test/fork_specific_items_unit.test.ts new file mode 100644 index 0000000000..0201958a8e --- /dev/null +++ b/cli/test/fork_specific_items_unit.test.ts @@ -0,0 +1,40 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { execSync } from "node:child_process"; +import os from "node:os"; +import path from "node:path"; + +import { inferWsNameFromForkBranch, type SyncOptions } from "../src/core/conf.ts"; + +// A fork id never has a wmill.yaml entry of its own. Unless it resolves to its +// parent's entry, a sync on the fork branch pushes the base files over the +// workspace-specific values the fork cloned from its parent. +test("a fork branch resolves to its parent's workspace entry", async () => { + const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_fork_specific_")); + const originalCwd = process.cwd(); + try { + execSync( + `git init -q -b wm-fork/dev-branch/t1 && git -c user.email=t@t -c user.name=t commit -q --allow-empty -m init`, + { cwd: repoDir }, + ); + process.chdir(repoDir); + + const config: SyncOptions = { + workspaces: { + dev: { + baseUrl: "http://localhost:8000/", + workspaceId: "dev-ws", + gitBranch: "dev-branch", + specificItems: { folders: ["f/team"] }, + }, + }, + }; + + expect(inferWsNameFromForkBranch(config, "wm-fork-t1")).toEqual("dev"); + // Only the fork the branch names inherits the parent's entry. + expect(inferWsNameFromForkBranch(config, "other-ws")).toBeUndefined(); + } finally { + process.chdir(originalCwd); + await rm(repoDir, { recursive: true, force: true }); + } +}); diff --git a/cli/test/preview.test.ts b/cli/test/preview.test.ts index ffa900c8f7..b0b00a2c7e 100644 --- a/cli/test/preview.test.ts +++ b/cli/test/preview.test.ts @@ -455,6 +455,12 @@ schema: // SCRIPT WITH MODULES PREVIEW TESTS // ============================================================================= +// Pinned: a release commit runs these tests while the same release publishes a new +// windmill-client, and the worker resolving that version before npm serves it fails the job. +// These are the only CLI tests running task/taskScript/workflow on a real client: bump the pin +// to a published release whenever the WAC client surface changes. +const WINDMILL_CLIENT = "windmill-client@1.819.0"; + test("script preview: script with modules (taskScript pattern)", async () => { await withTestBackend(async (backend, tempDir) => { await createWmillConfig(tempDir, { defaultTs: "bun" }); @@ -463,7 +469,7 @@ test("script preview: script with modules (taskScript pattern)", async () => { await createScript( tempDir, "f/test/wac_script.ts", - `import { task, taskScript, workflow } from "windmill-client"; + `import { task, taskScript, workflow } from "${WINDMILL_CLIENT}"; const helper = taskScript("./helper.ts"); @@ -512,7 +518,7 @@ test("script preview: script with modules (folder layout)", async () => { // Entry point script await writeFile( `${modDir}/script.ts`, - `import { task, taskScript, workflow } from "windmill-client"; + `import { task, taskScript, workflow } from "${WINDMILL_CLIENT}"; const helper = taskScript("./helper.ts"); diff --git a/cli/test/raw_app_sync.test.ts b/cli/test/raw_app_sync.test.ts index 315b274d80..33f1d3f6a6 100644 --- a/cli/test/raw_app_sync.test.ts +++ b/cli/test/raw_app_sync.test.ts @@ -61,8 +61,7 @@ root.render(); const PACKAGE_JSON = `{ "dependencies": { "react": "19.0.0", - "react-dom": "19.0.0", - "windmill-client": "^1" + "react-dom": "19.0.0" }, "devDependencies": { "@types/react-dom": "^19.0.0", diff --git a/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte b/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte index 1e094b97ce..380501f7b7 100644 --- a/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte +++ b/frontend/src/lib/components/apps/components/display/AppCustomComponent.svelte @@ -123,7 +123,13 @@ -
+ +
{#if !loaded && render} diff --git a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte index 4aff51d002..ca5e717891 100644 --- a/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte +++ b/frontend/src/lib/components/copilot/chat/RunScriptCard.svelte @@ -208,9 +208,12 @@ // Keyed by call id: a bare flag would carry one card's collapse onto the next message // reusing this instance. Open by default, since the run is what was asked for — // except for an inspection, which is usually a step in the reasoning rather than - // the answer, and which pays a fetch for being opened. + // the answer, and which pays a fetch for being opened. A run cancelled before it started + // is closed too: the row already says Cancelled, and there is no run to show. let toggled = $state<{ id: string; open: boolean } | undefined>(undefined) - const expanded = $derived(toggled?.id === message.tool_call_id ? toggled.open : !inspected) + const expanded = $derived( + toggled?.id === message.tool_call_id ? toggled.open : !inspected && (ran || !canceled) + ) $effect(() => { const target = inspected diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 655a872f96..ba718bf760 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -22,6 +22,8 @@ import { } from './modelConfig' import { applyReasoningToConfig, + completionsRejectsToolsWithReasoning, + explicitOffToken, requestsReasoning, stripLegacyThinkingSuffix, type ReasoningEffort @@ -69,6 +71,9 @@ interface AIProviderDetails { // the frontier model. The gpt-5 family is deprecated (retires 2026-12-11) but // still served, so it stays in the list below the 5.6 models. const OPENAI_MODELS = [ + 'gpt-6-sol', + 'gpt-6-astra', + 'gpt-6-luna', 'gpt-5.6-terra', 'gpt-5.6-sol', 'gpt-5.6-luna', @@ -87,7 +92,14 @@ export const AI_PROVIDERS: Record = { }, anthropic: { label: 'Anthropic', - defaultModels: ['claude-sonnet-5', 'claude-opus-5', 'claude-opus-4-8', 'claude-haiku-4-5'] + defaultModels: [ + 'claude-sonnet-5-5', + 'claude-opus-5-5', + 'claude-sonnet-5', + 'claude-opus-5', + 'claude-opus-4-8', + 'claude-haiku-4-5' + ] }, googleai: { label: 'Google AI', @@ -1160,6 +1172,12 @@ export async function getCompletion( // Use Completions API for other providers const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() + // gpt-5.5+ refuse function tools here unless reasoning is off, so the Responses API + // fallback turns it off where the model can rather than failing the turn. + const reasoningEffort = + tools?.length && completionsRejectsToolsWithReasoning(provider, modelProvider.model) + ? (explicitOffToken(provider, modelProvider.model) ?? options?.reasoningEffort) + : options?.reasoningEffort const completionConfig = applyReasoningToConfig( config.stream && STREAM_USAGE_PROVIDERS.has(provider) ? { @@ -1175,7 +1193,7 @@ export async function getCompletion( } : config, provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions', - options?.reasoningEffort + reasoningEffort ) const completion = client.chat.completions.create(completionConfig, { signal: abortController.signal, diff --git a/frontend/src/lib/components/copilot/modelConfig.test.ts b/frontend/src/lib/components/copilot/modelConfig.test.ts index 4f54eb223a..8bcdf70f54 100644 --- a/frontend/src/lib/components/copilot/modelConfig.test.ts +++ b/frontend/src/lib/components/copilot/modelConfig.test.ts @@ -21,6 +21,13 @@ describe('workspace context window overrides', () => { expect(getConfiguredModelContextWindow('openai', 'qwen-local', overrides)).toBeUndefined() expect(getEffectiveModelContextWindow('openai', 'qwen-local', overrides)).toBe(128_000) }) + + it('knows the gpt-6 and Claude 5.5 windows, so they do not compact at the assumed one', () => { + expect(getConfiguredModelContextWindow('openai', 'gpt-6-sol', undefined)).toBe(1_050_000) + expect(getConfiguredModelContextWindow('anthropic', 'claude-sonnet-5-5', undefined)).toBe( + 1_000_000 + ) + }) }) describe('usesAnthropicMessagesApi', () => { diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index 83e7771873..f9653bff25 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -58,7 +58,7 @@ export function usesOpenRouterPromptCaching(provider: AIProvider, model: string) // so it does not catch unrelated ids like Mistral's "open-mistral-*" or "optimus-*". export function requiresMaxCompletionTokens(model: string) { const baseModel = parseModelId(model).base - return baseModel.startsWith('gpt-5') || /^o\d/.test(baseModel) + return Number(/^gpt-(\d)(?:[.-]|$)/.exec(baseModel)?.[1] ?? 0) >= 5 || /^o\d/.test(baseModel) } // Context windows of the models we know, most specific entry first — the first @@ -87,6 +87,7 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [ ['claude', 200_000], // OpenAI — gpt-5 covers the base family (-mini / -nano) and the 5.1/5.2 // revisions, all 400K; 5.4/5.5 moved to 1M and 5.6 to 1.05M + ['gpt-6', 1_050_000], ['gpt-5.6', 1_050_000], ['gpt-5.5', 1_000_000], ['gpt-5.4', 1_000_000], @@ -140,6 +141,7 @@ const MODEL_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [ ['claude-fable', 64_000], ['claude-mythos', 64_000], // OpenAI + ['gpt-6', 128_000], ['gpt-5', 128_000], ['gpt-4.1', 32_768], ['gpt-4o', 16_384], diff --git a/frontend/src/lib/components/copilot/modelPricing.test.ts b/frontend/src/lib/components/copilot/modelPricing.test.ts index e295682187..4eded135ee 100644 --- a/frontend/src/lib/components/copilot/modelPricing.test.ts +++ b/frontend/src/lib/components/copilot/modelPricing.test.ts @@ -35,6 +35,16 @@ describe('resolveModelPrice', () => { expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined() }) + it('prices Claude 5.5 flat and leaves the tiered gpt-6 unpriced', () => { + expect(resolveModelPrice('anthropic', 'claude-opus-5-5', undefined)?.price).toMatchObject({ + input: 4, + output: 20, + cacheRead: 0.2 + }) + expect(resolveModelPrice('anthropic', 'claude-sonnet-5-5', undefined)?.price.input).toBe(2) + expect(resolveModelPrice('openai', 'gpt-6-sol', undefined)).toBeUndefined() + }) + it('reports an unknown model as unpriced rather than guessing', () => { expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined() }) diff --git a/frontend/src/lib/components/copilot/modelPricing.ts b/frontend/src/lib/components/copilot/modelPricing.ts index 1e5b63247b..b191a0bcaa 100644 --- a/frontend/src/lib/components/copilot/modelPricing.ts +++ b/frontend/src/lib/components/copilot/modelPricing.ts @@ -69,6 +69,8 @@ const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [ // fallback sits below the explicit entries rather than covering them. ['claude-fable-5', { input: 10, output: 50 }], ['claude-mythos-5', { input: 10, output: 50 }], + ['claude-opus-5-5', { input: 4, output: 20, cacheRead: 0.2 }], + ['claude-sonnet-5-5', { input: 2, output: 10, cacheRead: 0.2 }], ['claude-opus-5', { input: 5, output: 25 }], ['claude-opus-4-8', { input: 5, output: 25 }], ['claude-opus-4-7', { input: 5, output: 25 }], @@ -98,6 +100,9 @@ const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [ // Revisions past gpt-5 are priced separately by OpenAI and are not tracked here. // The matcher's revision guard already keeps them off the family rate; these // entries stay so a revision the guard admits still resolves to no rate. + // gpt-6 bills the whole request at a higher rate above 272K input tokens, which a + // per-model rate cannot express. + ['gpt-6', null], ['gpt-5.6', null], ['gpt-5.5', null], ['gpt-5.4', null], diff --git a/frontend/src/lib/components/copilot/reasoningParity.json b/frontend/src/lib/components/copilot/reasoningParity.json new file mode 100644 index 0000000000..93b8ac7eb2 --- /dev/null +++ b/frontend/src/lib/components/copilot/reasoningParity.json @@ -0,0 +1,84 @@ +[ + { + "provider": "anthropic", + "model": "claude-sonnet-5-5", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-opus-5-5", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-fable-5", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "aws_bedrock", + "model": "global.anthropic.claude-opus-5-5-v1:0", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-sonnet-5", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-opus-5-20260101", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { + "provider": "anthropic", + "model": "claude-opus-4-8", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { + "provider": "openai", + "model": "gpt-6-astra", + "canDisable": false, + "completionsToolsNeedOff": true + }, + { + "provider": "openai", + "model": "gpt-6-sol", + "canDisable": true, + "completionsToolsNeedOff": true + }, + { + "provider": "azure_openai", + "model": "gpt-6-luna", + "canDisable": true, + "completionsToolsNeedOff": true + }, + { + "provider": "openai", + "model": "gpt-5.6-sol", + "canDisable": true, + "completionsToolsNeedOff": true + }, + { "provider": "openai", "model": "gpt-5.5", "canDisable": true, "completionsToolsNeedOff": true }, + { "provider": "openai", "model": "gpt-5.7", "canDisable": true, "completionsToolsNeedOff": true }, + { + "provider": "openai", + "model": "gpt-5.1", + "canDisable": true, + "completionsToolsNeedOff": false + }, + { "provider": "openai", "model": "gpt-5", "canDisable": false, "completionsToolsNeedOff": false }, + { + "provider": "openai", + "model": "gpt-5-mini", + "canDisable": false, + "completionsToolsNeedOff": false + }, + { "provider": "openai", "model": "o3", "canDisable": false, "completionsToolsNeedOff": false } +] diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts index faf6f60264..30543e6c2f 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.test.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from 'vitest' import { applyReasoningToConfig, + completionsRejectsToolsWithReasoning, + explicitOffToken, getReasoningCapability, REASONING_OFF, resolveEffectiveReasoning, @@ -8,6 +10,8 @@ import { stripLegacyThinkingSuffix, supportsReasoning } from './reasoningRegistry' +import type { AIProvider } from '$lib/gen' +import parity from './reasoningParity.json' describe('stripLegacyThinkingSuffix', () => { it('removes the deprecated /thinking suffix', () => { @@ -261,6 +265,42 @@ describe('supportsReasoning (static registry)', () => { expect(getReasoningCapability('openrouter', 'x-ai/grok-4').canDisable).toBe(false) expect(getReasoningCapability('openrouter', 'deepseek/deepseek-r1').canDisable).toBe(false) }) + it('never sends a disable the 5.5 point releases and gpt-6-astra reject', () => { + // Live-verified: Claude 5.5 rejects `thinking: disabled`, gpt-6-astra rejects `none`. + for (const [provider, model] of [ + ['anthropic', 'claude-sonnet-5-5'], + ['anthropic', 'claude-opus-5-5'], + ['aws_bedrock', 'global.anthropic.claude-opus-5-5-v1:0'], + ['openai', 'gpt-6-astra'] + ] as const) { + expect(getReasoningCapability(provider, model).canDisable, model).toBe(false) + expect(explicitOffToken(provider, model), model).toBeUndefined() + } + expect(getReasoningCapability('openrouter', 'anthropic/claude-sonnet-5.5').canDisable).toBe( + false + ) + expect(explicitOffToken('openrouter', 'anthropic/claude-sonnet-5.5')).toBeUndefined() + // A dated Claude 5 id is not a point release. + expect(explicitOffToken('anthropic', 'claude-sonnet-5-20260101')).toBe('none') + expect(explicitOffToken('openai', 'gpt-6-sol')).toBe('none') + expect(getReasoningCapability('openai', 'gpt-6-luna')).toMatchObject({ + supported: true, + canDisable: true, + levels: ['low', 'medium', 'high', 'xhigh', 'max'] + }) + }) + it("reads Azure's gpt-35-turbo as gpt-3.5, not a gpt-5+ reasoning model", () => { + expect(supportsReasoning('azure_openai', 'gpt-35-turbo')).toBe(false) + expect(supportsReasoning('openai', 'gpt-35-turbo-16k')).toBe(false) + }) + it('finds the models that refuse function tools with reasoning on Chat Completions', () => { + for (const model of ['gpt-5.5', 'gpt-5.6-sol', 'gpt-6-astra']) { + expect(completionsRejectsToolsWithReasoning('openai', model), model).toBe(true) + } + for (const model of ['gpt-5', 'gpt-5.1', 'gpt-35-turbo', 'o3']) { + expect(completionsRejectsToolsWithReasoning('azure_openai', model), model).toBe(false) + } + }) it('forwards an explicit off as effort none through OpenRouter', () => { expect( resolveRequestReasoning({ @@ -353,6 +393,23 @@ describe('Azure AI Foundry reasoning follows the model family', () => { }) }) +describe('backend parity', () => { + // windmill-ai's `providers/mod.rs` test reads the same file. The backend rules see the + // model id alone, so the file holds only rows whose answer doesn't depend on the + // provider: a Bedrock- or Gemini-Pro-specific row belongs in the tests above. + it.each(parity)( + '$provider $model', + ({ provider, model, canDisable, completionsToolsNeedOff }) => { + const capability = getReasoningCapability(provider as AIProvider, model) + expect(capability.supported).toBe(true) + expect(capability.canDisable).toBe(canDisable) + expect(completionsRejectsToolsWithReasoning(provider as AIProvider, model)).toBe( + completionsToolsNeedOff + ) + } + ) +}) + describe('resolveEffectiveReasoning', () => { it('defaults capable models to high when unset', () => { expect(resolveEffectiveReasoning({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe( diff --git a/frontend/src/lib/components/copilot/reasoningRegistry.ts b/frontend/src/lib/components/copilot/reasoningRegistry.ts index 9915e9c42f..68934c22f3 100644 --- a/frontend/src/lib/components/copilot/reasoningRegistry.ts +++ b/frontend/src/lib/components/copilot/reasoningRegistry.ts @@ -1,5 +1,5 @@ import type { AIProvider, AIProviderModel } from '$lib/gen' -import { parseModelId, usesAnthropicMessagesApi } from './modelConfig' +import { usesAnthropicMessagesApi } from './modelConfig' /** * Reasoning effort is provider/model-specific. We never normalize a single @@ -33,11 +33,6 @@ export function stripLegacyThinkingSuffix(model: string): string { : model } -/** Bare model id without any provider/gateway prefix (e.g. OpenRouter's `openai/o3`). */ -function baseModelId(model: string): string { - return parseModelId(model).base -} - /** * Azure AI Foundry hosts multiple model families under one provider, so reasoning * support follows the underlying model rather than the provider: Claude deployments @@ -53,172 +48,218 @@ function reasoningProviderFamily(provider: AIProvider, model: string): AIProvide } /** - * Suggested effort levels per provider, sourced from each provider SDK's own - * vocabulary. + * Sentinel sent for the deepseek off case. It never reaches the wire as an + * effort: the 'deepseek' branch of `applyReasoningToConfig` translates it to + * the provider's `thinking: {type: "disabled"}` param (`reasoning_effort: + * "none"` is rejected by their API). */ -const PROVIDER_REASONING_LEVELS: Partial> = { - // DeepSeek accepts the full five-token vocabulary but only two levels are - // real: low/medium are server-mapped to high and xhigh to max — offering - // them would be a no-op knob. - deepseek: ['high', 'max'], - // Mistral's only effort token besides the 'none' disable is 'high' - // (anything else is rejected), so the knob is effectively on/off. - mistral: ['high'] -} +export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none' /** - * OpenRouter validates effort against its own vocabulary - * (minimal..xhigh + none) and translates per underlying provider, so the - * real ladder depends on the model family: Anthropic gets all five as - * distinct budget ratios (minimal 10% .. xhigh 95% of max_tokens); Gemini - * maps to thinkingLevel with xhigh clamped to high (a no-op vs high); - * OpenAI gets the token passed through verbatim, so the per-model OpenAI - * scoping applies; DeepSeek server-maps low/medium to high and xhigh to max. + * Sentinel for the Anthropic off case. Like the DeepSeek one it never reaches + * the wire as an effort: the 'anthropic' branch of `applyReasoningToConfig` + * translates it to `thinking: {type: "disabled"}`, the only off that Opus and + * Sonnet 5 respect. */ -function openrouterReasoningLevels(model: string): ReasoningEffort[] { - const m = model.toLowerCase() - const base = baseModelId(model) - if (/claude-(opus|sonnet)-(4|5)/.test(m)) { - return ['minimal', 'low', 'medium', 'high', 'xhigh'] - } - if (m.includes('gemini-')) { - return geminiReasoningLevels(m) - } - if (base.startsWith('gpt-5') || /^o\d/.test(base)) { - return openaiReasoningLevels(base) - } - if (m.includes('deepseek-v4')) { - return ['high', 'xhigh'] - } - return ['low', 'medium', 'high'] -} +export const ANTHROPIC_OFF_SENTINEL: ReasoningEffort = 'none' /** - * OpenAI's effort vocabulary is model-dependent: `minimal` exists on gpt-5 but - * not on gpt-5.1+, `xhigh` arrived on gpt-5.5 and `max` on gpt-5.6; o-series - * take low/medium/high. An unsupported level is rejected, so scope the list to - * the model. (`none` is the disable token, handled by `explicitOffToken`.) + * What the registry knows about one group of models. The rows below are matched in + * order against the lowercased model id, first match wins, so a narrower row goes above + * the row it overrides. A model no row matches does not reason, as far as we know. */ -function openaiReasoningLevels(model: string): ReasoningEffort[] { - const base = baseModelId(model) - if (/^gpt-5\.6/.test(base)) { - return ['low', 'medium', 'high', 'xhigh', 'max'] - } - if (/^gpt-5\.5/.test(base)) { - return ['low', 'medium', 'high', 'xhigh'] - } - if (/^gpt-5\./.test(base)) { - return ['low', 'medium', 'high'] - } - if (/^gpt-5/.test(base)) { - return ['minimal', 'low', 'medium', 'high'] - } - return ['low', 'medium', 'high'] +type ReasoningRule = { + match: RegExp + /** The levels the UI offers. An unsupported level is a 400, so this is per model. */ + levels: readonly ReasoningEffort[] + /** + * Whether "off" really stops the model reasoning. When false the UI offers no off: + * the provider would reject it, or coerce it to the lowest level. + */ + canDisable: boolean + /** + * The effort to send for off on a model that reasons when the field is omitted. + * Unset where omission is already off. Always `'none'`: `requestsReasoning` reads + * that as off, and each wire format translates it (see `applyReasoningToConfig`). + */ + offToken?: ReasoningEffort + /** + * gpt-5.5 and later refuse function tools on Chat Completions while they reason, + * even with the effort omitted (live-verified); the Responses API has no such limit. + */ + completionsToolsNeedOff?: boolean } +const LOW_TO_HIGH = ['low', 'medium', 'high'] +const LOW_TO_XHIGH = ['low', 'medium', 'high', 'xhigh'] +const LOW_TO_MAX = ['low', 'medium', 'high', 'xhigh', 'max'] +const MINIMAL_TO_HIGH = ['minimal', 'low', 'medium', 'high'] + /** - * Gemini's level ladder is model-dependent: Gemini 3+ Flash / Flash-Lite accept - * `minimal`, while 3.x Pro does not (and cannot disable thinking). Gemini 2.5 - * uses numeric budgets — the proxy maps the three tiers to budget values, so - * `minimal` is not offered there. + * Claude models whose thinking cannot be turned off: an explicit disable 400s. Fable, + * Mythos, and the 5.x point releases (Sonnet 5.5, Opus 5.5), whose lowest setting is + * adaptive thinking at `low` (live-verified). The version match stops at one digit so a + * dated id (`claude-sonnet-5-20260101`) stays Sonnet 5. */ -function geminiReasoningLevels(model: string): ReasoningEffort[] { - const m = model.toLowerCase() - const isGemini3Plus = !m.includes('gemini-2.5') - if (isGemini3Plus && (m.includes('flash') || m.includes('lite'))) { - return ['minimal', 'low', 'medium', 'high'] - } - return ['low', 'medium', 'high'] +const CLAUDE_ALWAYS_THINKING = /fable|mythos|claude-(opus|sonnet)-5[-.][1-9](?!\d)/ + +// Anthropic ids are matched anywhere in the id: Bedrock prefixes them +// (`us.anthropic.claude-opus-4-6-v1`). Opus 4.5 and older reject adaptive thinking. +const ANTHROPIC_RULES: ReasoningRule[] = [ + { match: CLAUDE_ALWAYS_THINKING, levels: LOW_TO_MAX, canDisable: false }, + // The 5 family thinks when the field is absent, so off is an explicit disable. + { + match: /claude-(opus|sonnet)-5/, + levels: LOW_TO_MAX, + canDisable: true, + offToken: ANTHROPIC_OFF_SENTINEL + }, + // 4.6-4.8 only think when asked, so omission is already off. + { match: /claude-opus-4-[78]/, levels: LOW_TO_MAX, canDisable: true }, + { match: /claude-(opus|sonnet)-4-6/, levels: ['low', 'medium', 'high', 'max'], canDisable: true } +] + +const BEDROCK_RULES: ReasoningRule[] = [ + ANTHROPIC_RULES[0], + // AWS documents Sonnet 5 on Bedrock as always thinking, where the native API + // accepts a disable for it. + { match: /claude-sonnet-5/, levels: LOW_TO_MAX, canDisable: false }, + ...ANTHROPIC_RULES.slice(1) +] + +// Anchored at the start or after a gateway's `vendor/`, and the major is one digit: +// Azure names gpt-3.5 `gpt-35-turbo`. `minimal` exists on gpt-5 only, `xhigh` from +// gpt-5.5, `max` from gpt-5.6. +const OPENAI_RULES: ReasoningRule[] = [ + // Live-verified: astra takes low..max only, where sol and luna also take `none`. + { + match: /(?:^|\/)gpt-6-astra/, + levels: LOW_TO_MAX, + canDisable: false, + completionsToolsNeedOff: true + }, + { + match: /(?:^|\/)gpt-[6-9](?:[.:-]|$)/, + levels: LOW_TO_MAX, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + { + match: /(?:^|\/)gpt-5\.6/, + levels: LOW_TO_MAX, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + { + match: /(?:^|\/)gpt-5\.5/, + levels: LOW_TO_XHIGH, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + // A later gpt-5 minor keeps the tools limit, which holds for every version from 5.5, + // but only the levels every gpt-5.x takes until it has a row of its own. + { + match: /(?:^|\/)gpt-5\.(?:[5-9]|\d{2,})/, + levels: LOW_TO_HIGH, + canDisable: true, + offToken: 'none', + completionsToolsNeedOff: true + }, + // gpt-5.1+ are off only through `none`: omitted, they reason at medium. + { match: /(?:^|\/)gpt-5\./, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' }, + // gpt-5 and the o-series reject `none` and reason when it is omitted. + { match: /(?:^|\/)gpt-5(?:[:-]|$)/, levels: MINIMAL_TO_HIGH, canDisable: false }, + { match: /(?:^|\/)o\d/, levels: LOW_TO_HIGH, canDisable: false } +] + +// Gemini 2.5/3 think by default; the backend proxy maps `none` to off on Flash, or to +// the floor on Pro, which enforces one (level `low` on 3.x, 128 tokens on 2.5). +// Gemini 3+ Flash / Flash-Lite accept `minimal`; 2.5 takes numeric budgets the proxy +// maps from three tiers. +const GEMINI_RULES: ReasoningRule[] = [ + { match: /gemini-2\.5.*pro/, levels: LOW_TO_HIGH, canDisable: false, offToken: 'none' }, + { match: /gemini-2\.5/, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' }, + { match: /gemini-3.*pro/, levels: LOW_TO_HIGH, canDisable: false, offToken: 'none' }, + { match: /gemini-3.*(flash|lite)/, levels: MINIMAL_TO_HIGH, canDisable: true, offToken: 'none' }, + { match: /gemini-3/, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' } +] + +const REASONING_RULES: Partial> = { + anthropic: ANTHROPIC_RULES, + aws_bedrock: BEDROCK_RULES, + openai: OPENAI_RULES, + azure_openai: OPENAI_RULES, + googleai: GEMINI_RULES, + deepseek: [ + // Every current API model takes reasoning_effort, but only two levels are real: + // low/medium are server-mapped to high, xhigh to max. The retired + // `deepseek-chat` alias means "non-thinking mode", so a saved selection on it + // must not silently become a thinking request. Off is a separate `thinking` param. + { + match: /(?:^|\/)deepseek(?!-chat(?::|$))/, + levels: ['high', 'max'], + canDisable: true, + offToken: DEEPSEEK_OFF_SENTINEL + } + ], + mistral: [ + // Only the ids verified to accept reasoning_effort (large, magistral, ministral + // and pinned versions reject it), whose only effort besides off is `high`. + { + match: /(?:^|\/)mistral-(?:(?:small|medium)-latest(?::|$)|medium-3[-.]5)/, + levels: ['high'], + canDisable: true + } + ], + // OpenRouter validates effort against its own vocabulary (minimal..xhigh + none) and + // translates it per underlying provider, which scopes the ladder: Anthropic gets all + // five as budget ratios, OpenAI gets the token verbatim, DeepSeek server-maps. `none` + // is its documented off, more reliable than omission; it can only disable a model + // whose upstream can. + openrouter: [ + { + match: /claude-(opus|sonnet)-5[-.][1-9](?!\d)/, + levels: ['minimal', ...LOW_TO_XHIGH], + canDisable: false + }, + { + match: /claude-(opus|sonnet)-(4|5)/, + levels: ['minimal', ...LOW_TO_XHIGH], + canDisable: true, + offToken: 'none' + }, + ...GEMINI_RULES, + ...OPENAI_RULES.map(({ completionsToolsNeedOff: _, ...rule }) => ({ + ...rule, + offToken: 'none' + })), + { match: /deepseek-v4/, levels: LOW_TO_XHIGH.slice(2), canDisable: true, offToken: 'none' }, + // deepseek-r1, grok-4 and :thinking variants reason unconditionally. + { + match: /deepseek-r|grok-4|:thinking/, + levels: LOW_TO_HIGH, + canDisable: false, + offToken: 'none' + } + ] } -/** - * Gemini Pro models cannot turn thinking off — the API enforces a floor - * (level `low` on 3.x Pro, a 128-token budget on 2.5 Pro), so an off option - * would silently mean "lowest". Flash / Flash-Lite can truly disable - * (budget 0 / level `minimal`). - */ -function geminiCanDisable(model: string): boolean { - return !model.toLowerCase().includes('pro') +/** The row that describes a model, and whether its provider family has rows at all. */ +function findReasoningRule( + provider: AIProvider, + model: string +): { rule: ReasoningRule | undefined; known: boolean } { + const rules = REASONING_RULES[reasoningProviderFamily(provider, model)] + const id = stripLegacyThinkingSuffix(model).toLowerCase() + return { rule: rules?.find((rule) => rule.match.test(id)), known: rules !== undefined } } -/** - * Anthropic's effort ladder is model-dependent: `xhigh` exists on Opus 4.7/4.8, - * the 5 family and Fable/Mythos; `max` also on Opus 4.6 and Sonnet 4.6. - * Offering an unsupported level would 400, so scope the list to the model. - */ -function anthropicReasoningLevels(model: string): ReasoningEffort[] { - const m = model.toLowerCase() - if ( - /claude-(opus|sonnet)-5/.test(m) || - /claude-opus-4-(7|8)/.test(m) || - m.includes('fable') || - m.includes('mythos') - ) { - return ['low', 'medium', 'high', 'xhigh', 'max'] - } - return ['low', 'medium', 'high', 'max'] -} - -/** Mistral writes both `mistral-medium-3.5` and `mistral-medium-3-5`. */ -function normalizeMistralId(model: string): string { - return baseModelId(model).replace(/\./g, '-') -} - -/** - * Conservative static predicate for whether a model accepts an effort knob. - * Kept tight to avoid 400s on models that reject reasoning params. - */ -function supportsReasoningStatic(provider: AIProvider, model: string): boolean { - const m = model.toLowerCase() - const base = baseModelId(model) - switch (reasoningProviderFamily(provider, model)) { - case 'anthropic': - // Bedrock serves the same Claude models under prefixed ids - // (e.g. `us.anthropic.claude-opus-4-6-v1`), so match on the full string. - case 'aws_bedrock': - // 4.6+ only: Opus 4.5 rejects adaptive thinking (and, on Bedrock, - // the whole output_config surface) — live-verified hard 400. - return ( - /claude-opus-(4-(6|7|8)|5)/.test(m) || - /claude-sonnet-(4-6|5)/.test(m) || - m.includes('fable') || - m.includes('mythos') - ) - case 'openai': - case 'azure_openai': - return base.startsWith('gpt-5') || /^o\d/.test(base) - case 'openrouter': - // Best-effort markers for models whose `supported_parameters` include - // `reasoning` in OpenRouter's catalog; OpenRouter translates the effort - // per underlying provider. - return ( - base.startsWith('gpt-5') || - /^o\d/.test(base) || - /claude-(opus|sonnet)-(4|5)/.test(m) || - /gemini-(2\.5|3)/.test(m) || - m.includes('deepseek-r') || - m.includes('deepseek-v4') || - m.includes('grok-4') || - m.includes(':thinking') - ) - case 'googleai': - return /gemini-(2\.5|3)/.test(m) - case 'deepseek': - // All current API models take reasoning_effort (live-verified). The - // retired `deepseek-chat` alias stays excluded: its documented meaning - // is "non-thinking mode", so a saved selection on it must not silently - // become a thinking request. - return base.startsWith('deepseek') && base !== 'deepseek-chat' - case 'mistral': - // Only the ids verified to accept reasoning_effort; other models - // (large, magistral, ministral, pinned versions) reject the param. - return ( - /^mistral-(small|medium)-latest$/.test(base) || - normalizeMistralId(model).startsWith('mistral-medium-3-5') - ) - default: - return false - } +/** Whether Chat Completions needs reasoning off for this model to take function tools. */ +export function completionsRejectsToolsWithReasoning(provider: AIProvider, model: string): boolean { + return findReasoningRule(provider, model).rule?.completionsToolsNeedOff ?? false } export type ReasoningCapability = { @@ -241,89 +282,12 @@ export type ReasoningCapability = { known: boolean } -/** Provider families the registry has real rules for; everything else is a shrug. */ -const KNOWN_REASONING_FAMILIES: ReadonlySet = new Set([ - 'anthropic', - 'aws_bedrock', - 'openai', - 'azure_openai', - 'openrouter', - 'googleai', - 'deepseek', - 'mistral' -]) - /** Resolve the reasoning capability of a model from the static registry. */ export function getReasoningCapability(provider: AIProvider, model: string): ReasoningCapability { - const bareModel = stripLegacyThinkingSuffix(model) - const known = KNOWN_REASONING_FAMILIES.has(reasoningProviderFamily(provider, bareModel)) - const supported = supportsReasoningStatic(provider, bareModel) - if (!supported) { - return { supported: false, levels: [], canDisable: false, known } - } - const family = reasoningProviderFamily(provider, bareModel) - const levels = - family === 'anthropic' || family === 'aws_bedrock' - ? anthropicReasoningLevels(bareModel) - : family === 'googleai' - ? geminiReasoningLevels(bareModel) - : family === 'openai' || family === 'azure_openai' - ? openaiReasoningLevels(bareModel) - : family === 'openrouter' - ? openrouterReasoningLevels(bareModel) - : (PROVIDER_REASONING_LEVELS[family] ?? ['low', 'medium', 'high']) - return { supported, levels, canDisable: canDisableReasoning(provider, bareModel), known } -} - -/** - * Whether selecting "off" truly disables reasoning for the model. Off is - * sent either as an explicit provider disable (see `explicitOffToken`) or by - * omitting the effort — which only works where the model doesn't reason by - * default. - */ -function canDisableReasoning(provider: AIProvider, model: string): boolean { - const m = model.toLowerCase() - const base = baseModelId(model) - switch (reasoningProviderFamily(provider, model)) { - case 'anthropic': - // Every Claude but Fable and Mythos can stop thinking: 4.6-4.8 by - // omission, and the 5 family through the explicit disable that - // `explicitOffToken` sends. - return !ANTHROPIC_ALWAYS_THINKING.test(m) - case 'aws_bedrock': - // Same models, different answer: AWS documents Sonnet 5 on Bedrock as - // always thinking, where the native API accepts a disable for it. - return !(ANTHROPIC_ALWAYS_THINKING.test(m) || m.includes('claude-sonnet-5')) - case 'googleai': - return geminiCanDisable(model) - case 'openai': - case 'azure_openai': - // gpt-5.1+ accept effort 'none'; gpt-5 and o-series reject it and - // reason at `medium` by default, so omission isn't off either. - return /^gpt-5\./.test(base) - case 'openrouter': - // 'none' is in OpenRouter's vocabulary, but the gateway can't - // disable a model whose upstream can't — scope off per underlying - // family, like the levels. - // The 5 family thinks by default, but its upstream takes an explicit - // disable, so the gateway's 'none' has something to translate to. - if (/claude-(opus|sonnet)-(4|5)/.test(m)) { - return true - } - if (m.includes('gemini-')) { - return geminiCanDisable(m) - } - if (base.startsWith('gpt-5') || /^o\d/.test(base)) { - return /^gpt-5\./.test(base) - } - if (m.includes('deepseek-v4')) { - return true - } - // grok-4, deepseek-r1 and :thinking variants reason unconditionally. - return false - default: - return true - } + const { rule, known } = findReasoningRule(provider, model) + return rule + ? { supported: true, levels: [...rule.levels], canDisable: rule.canDisable, known } + : { supported: false, levels: [], canDisable: false, known } } export function supportsReasoning(provider: AIProvider, model: string): boolean { @@ -352,64 +316,14 @@ export function resolveEffectiveReasoning( : undefined } -/** - * Sentinel sent for the deepseek off case. It never reaches the wire as an - * effort: the 'deepseek' branch of `applyReasoningToConfig` translates it to - * the provider's `thinking: {type: "disabled"}` param (`reasoning_effort: - * "none"` is rejected by their API). - */ -export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none' - -/** - * Sentinel for the Anthropic off case. Like the DeepSeek one it never reaches - * the wire as an effort: the 'anthropic' branch of `applyReasoningToConfig` - * translates it to `thinking: {type: "disabled"}`, which is the only off the - * always-on 5 family respects. - */ -export const ANTHROPIC_OFF_SENTINEL: ReasoningEffort = 'none' - -/** Claude models whose thinking cannot be turned off — an explicit disable 400s. */ -const ANTHROPIC_ALWAYS_THINKING = /fable|mythos/ - /** * Disable token to forward when the user explicitly turns reasoning off on a * model that reasons *by default* — omitting the field would silently keep - * the default-on behavior. Undefined means omission is the correct off. Every - * token is `'none'`: `requestsReasoning` reads that value as off. + * the default-on behavior. Undefined means omission is the correct off, or that + * the model cannot be turned off at all. */ export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined { - switch (reasoningProviderFamily(provider, model)) { - case 'anthropic': - // Claude 4.6-4.8 only think when asked, so omission is already a - // real off there and stays the wire form. Only the 5 family, which - // thinks when the field is absent, needs the explicit disable — - // Fable and Mythos reject it outright and get no off token at all. - return /claude-(opus|sonnet)-5/.test(model.toLowerCase()) ? ANTHROPIC_OFF_SENTINEL : undefined - case 'aws_bedrock': - // Bedrock's Sonnet 5 cannot be disabled at all, so only Opus 5 gets - // the sentinel; the rest keep omission. - return model.toLowerCase().includes('claude-opus-5') ? ANTHROPIC_OFF_SENTINEL : undefined - case 'googleai': - // Gemini 2.5/3 think by default (dynamic budget / level). The backend - // proxy maps 'none' to off on Flash, or the floor on Pro (only - // reachable via a stale persisted preference — see canDisableReasoning). - return 'none' - case 'deepseek': - return DEEPSEEK_OFF_SENTINEL - case 'openai': - case 'azure_openai': - // gpt-5.1+ reasoning is off only via the explicit 'none' effort - // (gpt-5.5 defaults to medium when the field is omitted). - return /^gpt-5\./.test(baseModelId(model)) ? 'none' : undefined - case 'openrouter': - // OpenRouter validates effort against xhigh..minimal|none and - // documents 'none' as disabling reasoning, translated per the - // underlying provider — more reliable than omission, which keeps - // reasoning-by-default models thinking. - return 'none' - default: - return undefined - } + return findReasoningRule(provider, model).rule?.offToken } /** diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 831380748e..2c46c48b2c 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -1,6 +1,6 @@ { "gitSyncTest": "hub/28950/git-repo-test-read-write-windmill", - "gitInitRepo": "hub/28970/git-sync-init-repository-windmill", + "gitInitRepo": "hub/28993/git-sync-init-repository-windmill", "slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack", "emailErrorHandler": "hub/19795/workspace-or-error-handler-email", "slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack",