mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-13 00:03:00 +00:00
Merge branch 'agents-home-plus-menu' into agents-run-on-behalf-of
This commit is contained in:
hugocasa
@@ -0,0 +1,171 @@
|
||||
name: AI Evals (scheduled)
|
||||
|
||||
# Full ai_evals suites on current models, posted to the AI evals dashboard in
|
||||
# the windmill-prod workspace (f/ai/ai_evals_dashboard) so quality is tracked
|
||||
# over time. Weekly, since one pass of every suite at --runs 3 costs about 50M
|
||||
# tokens per model: every suite runs on MODELS, and global (the mode users get)
|
||||
# also runs on GLOBAL_EXTRA_MODELS, one flagship per other provider. Run it by
|
||||
# hand to measure a branch against main.
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 3 * * 1"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
modes:
|
||||
description: "Space-separated modes"
|
||||
default: "global flow app script cli"
|
||||
models:
|
||||
description: "Space-separated model aliases (bun run cli -- models)"
|
||||
default: "sonnet-5.5"
|
||||
global_extra_models:
|
||||
description: "Extra model aliases for global mode only"
|
||||
default: "gpt-6-astra gemini-3.8-flash"
|
||||
runs:
|
||||
description: "Runs per case"
|
||||
default: "3"
|
||||
reasoning:
|
||||
description: "Reasoning effort for frontend modes (empty: the product default)"
|
||||
default: ""
|
||||
|
||||
concurrency:
|
||||
group: ai-evals-scheduled-${{ github.ref }}
|
||||
|
||||
env:
|
||||
MODELS: ${{ inputs.models || 'sonnet-5.5' }}
|
||||
GLOBAL_EXTRA_MODELS: ${{ inputs.global_extra_models || 'gpt-6-astra gemini-3.8-flash' }}
|
||||
RUNS: ${{ inputs.runs || '3' }}
|
||||
REASONING: ${{ inputs.reasoning }}
|
||||
INGEST_URL: https://app.windmill.dev/api/r/f/ai/ingest_ai_eval_run
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
modes: ${{ steps.modes.outputs.modes }}
|
||||
steps:
|
||||
- id: modes
|
||||
env:
|
||||
MODES: ${{ inputs.modes || 'global flow app script cli' }}
|
||||
run: |
|
||||
echo "modes=$(jq -cn --arg m "$MODES" '$m | split(" ")
|
||||
| map(select(IN("global", "flow", "app", "script", "cli")))')" >> "$GITHUB_OUTPUT"
|
||||
|
||||
evals:
|
||||
needs: setup
|
||||
runs-on: ubicloud-standard-16
|
||||
timeout-minutes: 330
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: ${{ fromJSON(needs.setup.outputs.modes) }}
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16
|
||||
ports:
|
||||
- 5432:5432
|
||||
env:
|
||||
POSTGRES_DB: windmill
|
||||
POSTGRES_PASSWORD: changeme
|
||||
options: >-
|
||||
--health-cmd pg_isready --health-interval 10s --health-timeout 5s
|
||||
--health-retries 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions-rust-lang/setup-rust-toolchain@v1
|
||||
with:
|
||||
cache-workspaces: backend
|
||||
toolchain: 1.97.0
|
||||
|
||||
- uses: oven-sh/setup-bun@v2
|
||||
with:
|
||||
bun-version: 1.4.0
|
||||
|
||||
- uses: actions/setup-node@v7
|
||||
with:
|
||||
node-version: "24"
|
||||
|
||||
- name: Build Windmill
|
||||
working-directory: ./backend
|
||||
env:
|
||||
SQLX_OFFLINE: true
|
||||
CARGO_BUILD_JOBS: 12
|
||||
RUSTFLAGS: ""
|
||||
run: cargo build --features quickjs
|
||||
|
||||
- name: Start Windmill
|
||||
working-directory: ./backend
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill
|
||||
RUST_LOG: info
|
||||
run: |
|
||||
mkdir -p ../ai_evals/logs
|
||||
./target/debug/windmill > ../ai_evals/logs/windmill.log 2>&1 &
|
||||
for i in $(seq 1 60); do
|
||||
curl -sf http://localhost:8000/api/version > /dev/null 2>&1 && break
|
||||
sleep 2
|
||||
done
|
||||
curl -sf http://localhost:8000/api/version > /dev/null || { tail -50 ../ai_evals/logs/windmill.log; exit 1; }
|
||||
|
||||
- name: Install frontend deps + generate client
|
||||
working-directory: ./frontend
|
||||
run: |
|
||||
npm ci
|
||||
npm run generate-backend-client
|
||||
|
||||
- name: Install CLI deps + generate CLI client
|
||||
working-directory: ./cli
|
||||
run: bun install && ./gen_wm_client.sh && ./windmill-utils-internal/gen_wm_client.sh
|
||||
|
||||
- name: Run ${{ matrix.mode }} evals
|
||||
working-directory: ./ai_evals
|
||||
env:
|
||||
MODE: ${{ matrix.mode }}
|
||||
WMILL_AI_EVAL_BACKEND_URL: http://localhost:8000
|
||||
WMILL_AI_EVAL_BACKEND_WORKSPACE: integration-tests
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
GEMINI_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
||||
run: |
|
||||
bun install
|
||||
mkdir -p results
|
||||
models="$MODELS"
|
||||
[ "$MODE" = global ] && models="$models $GLOBAL_EXTRA_MODELS"
|
||||
reasoning=()
|
||||
[ -n "$REASONING" ] && [ "$MODE" != cli ] && reasoning=(--reasoning "$REASONING")
|
||||
for m in $models; do
|
||||
bun run cli -- run "$MODE" --model "$m" --runs "$RUNS" "${reasoning[@]}" \
|
||||
--output "$PWD/results/$MODE-$m.json" || echo "::warning::$MODE on $m errored"
|
||||
done
|
||||
|
||||
- name: Post results to the dashboard
|
||||
if: always()
|
||||
working-directory: ./ai_evals
|
||||
env:
|
||||
MODE: ${{ matrix.mode }}
|
||||
INGEST_TOKEN: ${{ secrets.AI_EVALS_INGEST_TOKEN }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
[ -n "$INGEST_TOKEN" ] || { echo "::warning::AI_EVALS_INGEST_TOKEN is not set"; exit 0; }
|
||||
shopt -s nullglob
|
||||
for f in results/"$MODE"-*.json; do
|
||||
# Keep what the dashboard reads; the full traces stay in the run's artifacts.
|
||||
jq -c --arg ref "$GITHUB_REF" --arg trigger "$GITHUB_EVENT_NAME" --arg url "$RUN_URL" '
|
||||
{result_json: (del(.cases[].prompt, .cases[].initialPath, .cases[].expectedPath)
|
||||
| .cases[].attempts[] |= {attempt, passed, durationMs, toolCallCount, judgeScore,
|
||||
judgeSummary, error, tokenUsage, checks: [.checks[]? | {name, passed}]}),
|
||||
git_ref: $ref, trigger: $trigger, run_url: $url}' "$f" \
|
||||
| curl -sf --retry 3 -X POST "$INGEST_URL" \
|
||||
-H "Authorization: Bearer $INGEST_TOKEN" -H 'content-type: application/json' \
|
||||
--data-binary @- && echo " <- $f" || echo "::warning::failed to post $f"
|
||||
done
|
||||
|
||||
- name: Archive logs and results
|
||||
uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: ai-evals-${{ matrix.mode }}
|
||||
path: |
|
||||
ai_evals/logs
|
||||
ai_evals/results
|
||||
+3
-14
@@ -77,31 +77,20 @@ Public CLI surface:
|
||||
- `--verbose`: stream assistant output for frontend runs
|
||||
- `--skip-judge`: skip LLM judge scoring for the run
|
||||
- `--execution-only`: only require the model/proxy/frontend loop to complete; skip validators, tool expectations, backend artifact validation, and judge scoring
|
||||
- `--reasoning <effort>`: reasoning effort for frontend modes (`off`, `low`, `medium`, `high`, `max`, …); without it the product's default applies (`high` on models that can reason). The effort is appended to the recorded model label (`anthropic:claude-sonnet-5-5@max`)
|
||||
- `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl` for full-suite runs only
|
||||
- `--backend-validation <mode>`: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals
|
||||
|
||||
## Models
|
||||
|
||||
Use `bun run cli -- models` to see the current aliases.
|
||||
|
||||
Today:
|
||||
|
||||
- `haiku`
|
||||
- `sonnet`
|
||||
- `opus`
|
||||
- `4o`
|
||||
- `gpt-5.5`
|
||||
- `gemini-3-flash-preview`
|
||||
- `gemini-3.1-pro-preview`
|
||||
- `deepseek-v4-flash`
|
||||
- `deepseek-v4-pro`
|
||||
Use `bun run cli -- models` to see the current aliases; `core/models.ts` is the list.
|
||||
|
||||
Notes:
|
||||
|
||||
- the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
|
||||
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
|
||||
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
|
||||
- the judge model is separate and currently defaults to `claude-sonnet-4-6`; use `--skip-judge` for deterministic-only runs
|
||||
- the judge model is separate and currently defaults to `claude-sonnet-5-5`; use `--skip-judge` for deterministic-only runs
|
||||
|
||||
## Case Format
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import { runEval } from "../shared";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { TokenUsage, ToolCallDetail } from "../shared/types";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
import { evalReasoningEffort } from "../shared/providerConfig";
|
||||
|
||||
export interface ScriptEvalResult {
|
||||
success: boolean;
|
||||
@@ -41,14 +42,15 @@ export interface ScriptEvalOptions {
|
||||
function resolveModelProvider(
|
||||
model: string,
|
||||
provider?: AIProvider,
|
||||
): AIProviderModel {
|
||||
): AIProviderModel & { reasoning?: string } {
|
||||
const reasoning = evalReasoningEffort();
|
||||
if (provider) {
|
||||
return { provider, model };
|
||||
return { provider, model, reasoning };
|
||||
}
|
||||
if (model.startsWith("claude")) {
|
||||
return { provider: "anthropic", model };
|
||||
return { provider: "anthropic", model, reasoning };
|
||||
}
|
||||
return { provider: "openai", model };
|
||||
return { provider: "openai", model, reasoning };
|
||||
}
|
||||
|
||||
export async function runScriptEval(
|
||||
|
||||
@@ -12,6 +12,12 @@ export interface EvalClients {
|
||||
export interface ResolvedEvalModelProvider {
|
||||
provider: FrontendEvalProvider;
|
||||
model: string;
|
||||
reasoning?: string;
|
||||
}
|
||||
|
||||
/** `run --reasoning` hands the effort to the frontend runtime through the environment. */
|
||||
export function evalReasoningEffort(): string | undefined {
|
||||
return process.env.WMILL_AI_EVAL_REASONING || undefined;
|
||||
}
|
||||
|
||||
export interface WindmillAiProxyClientConfig {
|
||||
@@ -73,6 +79,13 @@ export function createEvalClients(input: {
|
||||
export function resolveEvalModelProvider(
|
||||
model: string,
|
||||
provider?: FrontendEvalProvider,
|
||||
): ResolvedEvalModelProvider {
|
||||
return { ...resolveProvider(model, provider), reasoning: evalReasoningEffort() };
|
||||
}
|
||||
|
||||
function resolveProvider(
|
||||
model: string,
|
||||
provider?: FrontendEvalProvider,
|
||||
): ResolvedEvalModelProvider {
|
||||
if (provider) {
|
||||
return { provider, model };
|
||||
|
||||
+29
-91
@@ -5,8 +5,8 @@
|
||||
"": {
|
||||
"name": "windmill-ai-evals",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.25",
|
||||
"@anthropic-ai/sdk": "^0.39.0",
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.3.284",
|
||||
"@anthropic-ai/sdk": "^0.129.0",
|
||||
"commander": "^14.0.3",
|
||||
"openai": "^6.9.1",
|
||||
"yaml": "^2.8.3",
|
||||
@@ -18,66 +18,44 @@
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.2.87", "", { "dependencies": { "@anthropic-ai/sdk": "^0.74.0", "@modelcontextprotocol/sdk": "^1.27.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "^0.34.2", "@img/sharp-darwin-x64": "^0.34.2", "@img/sharp-linux-arm": "^0.34.2", "@img/sharp-linux-arm64": "^0.34.2", "@img/sharp-linux-x64": "^0.34.2", "@img/sharp-linuxmusl-arm64": "^0.34.2", "@img/sharp-linuxmusl-x64": "^0.34.2", "@img/sharp-win32-arm64": "^0.34.2", "@img/sharp-win32-x64": "^0.34.2" }, "peerDependencies": { "zod": "^4.0.0" } }, "sha512-WWmgBPxPhBOvNT0ujI8vPTI2lK+w5YEkEZ/y1mH0EDkK/0kBnxVJNhCtG5vnueiAViwLoUOFn66pbkDiivijdA=="],
|
||||
"@anthropic-ai/claude-agent-sdk": ["@anthropic-ai/claude-agent-sdk@0.3.284", "", { "optionalDependencies": { "@anthropic-ai/claude-agent-sdk-darwin-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-darwin-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-arm64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64": "0.3.284", "@anthropic-ai/claude-agent-sdk-linux-x64-musl": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-arm64": "0.3.284", "@anthropic-ai/claude-agent-sdk-win32-x64": "0.3.284" }, "peerDependencies": { "@anthropic-ai/sdk": ">=0.93.0", "@modelcontextprotocol/sdk": "^1.29.0", "zod": "^4.0.0" } }, "sha512-NSoJwEq6nFSf8dtaacYx37QdGgqApI3eHFUlxclMLYi8irb6ZJwEaUnPnLUCyAyg9W/tMkgZC0GXWLRCc01I0w=="],
|
||||
|
||||
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.39.0", "", { "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7" } }, "sha512-eMyDIPRZbt1CCLErRCi3exlAvNkBtRe+kW5vvJyef93PmNr/clstYgHhtvmkxN82nlKgzyGPCyGxrm0JQ1ZIdg=="],
|
||||
"@anthropic-ai/claude-agent-sdk-darwin-arm64": ["@anthropic-ai/claude-agent-sdk-darwin-arm64@0.3.284", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gKY9MUjY83398uCiPLHsd87kyzu7agIM7ApqWpJkpSINep6hxx4rNoR8bUbNWg49Aoe/PW/DJkQByAQuNzR9rg=="],
|
||||
|
||||
"@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="],
|
||||
"@anthropic-ai/claude-agent-sdk-darwin-x64": ["@anthropic-ai/claude-agent-sdk-darwin-x64@0.3.284", "", { "os": "darwin", "cpu": "x64" }, "sha512-P+q6Z7sKeYz99uE7RJc4au1IK+4SiWe69pyHY3QXMnjZq8hSFU/5Mo8iSurLk9jRlihsMltV1rLA5naouYyCAQ=="],
|
||||
|
||||
"@anthropic-ai/claude-agent-sdk-linux-arm64": ["@anthropic-ai/claude-agent-sdk-linux-arm64@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-LDpuYDaz+pCdG29iy1pw4P1D2YVtpZOb4rWOPxxSpg2fFyUeQ5WrfnuszOxhDWpNh1k1ekDiKm3A9OzGcrkdtA=="],
|
||||
|
||||
"@anthropic-ai/claude-agent-sdk-linux-arm64-musl": ["@anthropic-ai/claude-agent-sdk-linux-arm64-musl@0.3.284", "", { "os": "linux", "cpu": "arm64" }, "sha512-U3/uAv1TS8sMWsgWQSxYbdND5AaRXXpP5RxHU1oYC5ZizlZE8/xAVum9dj1QX3rVZJg8EzsqZveq/AkEv5ABfQ=="],
|
||||
|
||||
"@anthropic-ai/claude-agent-sdk-linux-x64": ["@anthropic-ai/claude-agent-sdk-linux-x64@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-yGytBCCwJvWeg1FzFpgnLM9BOM2vKVExtv6pMJHMtF82J5yhKHcxodRaYlQVd/HzJJ2nruKb9EnrhRBZ//4yRw=="],
|
||||
|
||||
"@anthropic-ai/claude-agent-sdk-linux-x64-musl": ["@anthropic-ai/claude-agent-sdk-linux-x64-musl@0.3.284", "", { "os": "linux", "cpu": "x64" }, "sha512-4x0Q8CFkCNbeENE7pRhuUmkwh/sQfTq1yWi4ZE56stve9pgoKzu6ZDa1SEtvK1YIooGE/qM5rVS8xFLE1gbQ0w=="],
|
||||
|
||||
"@anthropic-ai/claude-agent-sdk-win32-arm64": ["@anthropic-ai/claude-agent-sdk-win32-arm64@0.3.284", "", { "os": "win32", "cpu": "arm64" }, "sha512-zOXkdPHhElyxyFJ6eY5R+YGUswqBDZ+ZHNaRmQGJ28RkQYgwG5imNy/nfFR+/CPGlMnH9ZzbGc2rKYApm1K4Tg=="],
|
||||
|
||||
"@anthropic-ai/claude-agent-sdk-win32-x64": ["@anthropic-ai/claude-agent-sdk-win32-x64@0.3.284", "", { "os": "win32", "cpu": "x64" }, "sha512-VGaFRDCOPloj5IjvJTyy5JsSl9sE2HR3W+A6f/cjgh23/7Y5vK7/ka+1JQD+AkoGO9tbHqP2O6god3IBrLvOaw=="],
|
||||
|
||||
"@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.129.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1", "standardwebhooks": "^1.0.0" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-MH7LB20kNpGLUpPTe2OG5XvL/KhX8HUO9XyBaFjgFk6TLS4Xjt0VPIuMKywe7POKKPslX+QxSlVok9e+8kG5Nw=="],
|
||||
|
||||
"@babel/runtime": ["@babel/runtime@7.29.7", "", {}, "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw=="],
|
||||
|
||||
"@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="],
|
||||
|
||||
"@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="],
|
||||
|
||||
"@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="],
|
||||
|
||||
"@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="],
|
||||
|
||||
"@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="],
|
||||
|
||||
"@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="],
|
||||
|
||||
"@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="],
|
||||
|
||||
"@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="],
|
||||
|
||||
"@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="],
|
||||
|
||||
"@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="],
|
||||
|
||||
"@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="],
|
||||
|
||||
"@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="],
|
||||
|
||||
"@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="],
|
||||
|
||||
"@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="],
|
||||
|
||||
"@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="],
|
||||
|
||||
"@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="],
|
||||
|
||||
"@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="],
|
||||
|
||||
"@modelcontextprotocol/sdk": ["@modelcontextprotocol/sdk@1.29.0", "", { "dependencies": { "@hono/node-server": "^1.19.9", "ajv": "^8.17.1", "ajv-formats": "^3.0.1", "content-type": "^1.0.5", "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", "eventsource-parser": "^3.0.0", "express": "^5.2.1", "express-rate-limit": "^8.2.1", "hono": "^4.11.4", "jose": "^6.1.3", "json-schema-typed": "^8.0.2", "pkce-challenge": "^5.0.0", "raw-body": "^3.0.0", "zod": "^3.25 || ^4.0", "zod-to-json-schema": "^3.25.1" }, "peerDependencies": { "@cfworker/json-schema": "^4.1.1" }, "optionalPeers": ["@cfworker/json-schema"] }, "sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ=="],
|
||||
|
||||
"@stablelib/base64": ["@stablelib/base64@1.0.1", "", {}, "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.11", "", { "dependencies": { "bun-types": "1.3.11" } }, "sha512-5vPne5QvtpjGpsGYXiFyycfpDF2ECyPcTSsFBMa0fraoxiQyMJ3SmuQIGhzPg2WJuWxVBoxWJ2kClYTcw/4fAg=="],
|
||||
|
||||
"@types/node": ["@types/node@18.19.130", "", { "dependencies": { "undici-types": "~5.26.4" } }, "sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg=="],
|
||||
|
||||
"@types/node-fetch": ["@types/node-fetch@2.6.13", "", { "dependencies": { "@types/node": "*", "form-data": "^4.0.4" } }, "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw=="],
|
||||
|
||||
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
|
||||
"@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
|
||||
|
||||
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
|
||||
|
||||
"agentkeepalive": ["agentkeepalive@4.6.0", "", { "dependencies": { "humanize-ms": "^1.2.1" } }, "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ=="],
|
||||
|
||||
"ajv": ["ajv@8.18.0", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A=="],
|
||||
|
||||
"ajv-formats": ["ajv-formats@3.0.1", "", { "dependencies": { "ajv": "^8.0.0" } }, "sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ=="],
|
||||
|
||||
"asynckit": ["asynckit@0.4.0", "", {}, "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="],
|
||||
|
||||
"body-parser": ["body-parser@2.2.2", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.3", "http-errors": "^2.0.0", "iconv-lite": "^0.7.0", "on-finished": "^2.4.1", "qs": "^6.14.1", "raw-body": "^3.0.1", "type-is": "^2.0.1" } }, "sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-1KGPpoxQWl9f6wcZh57LvrPIInQMn2TQ7jsgxqpRzg+l0QPOFvJVH7HmvHo/AiPgwXy+/Thf6Ov3EdVn1vOabg=="],
|
||||
@@ -88,8 +66,6 @@
|
||||
|
||||
"call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="],
|
||||
|
||||
"combined-stream": ["combined-stream@1.0.8", "", { "dependencies": { "delayed-stream": "~1.0.0" } }, "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg=="],
|
||||
|
||||
"commander": ["commander@14.0.3", "", {}, "sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw=="],
|
||||
|
||||
"content-disposition": ["content-disposition@1.0.1", "", {}, "sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q=="],
|
||||
@@ -106,8 +82,6 @@
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"delayed-stream": ["delayed-stream@1.0.0", "", {}, "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="],
|
||||
|
||||
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
|
||||
|
||||
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
|
||||
@@ -122,14 +96,10 @@
|
||||
|
||||
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
|
||||
|
||||
"es-set-tostringtag": ["es-set-tostringtag@2.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "get-intrinsic": "^1.2.6", "has-tostringtag": "^1.0.2", "hasown": "^2.0.2" } }, "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA=="],
|
||||
|
||||
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
|
||||
|
||||
"etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
|
||||
|
||||
"event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="],
|
||||
|
||||
"eventsource": ["eventsource@3.0.7", "", { "dependencies": { "eventsource-parser": "^3.0.1" } }, "sha512-CRT1WTyuQoD771GW56XEZFQ/ZoSfWid1alKGDYMmkt2yl8UXrVR4pspqWNEcqKvVIzg6PAltWjxcSSPrboA4iA=="],
|
||||
|
||||
"eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="],
|
||||
@@ -140,16 +110,12 @@
|
||||
|
||||
"fast-deep-equal": ["fast-deep-equal@3.1.3", "", {}, "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q=="],
|
||||
|
||||
"fast-sha256": ["fast-sha256@1.3.0", "", {}, "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ=="],
|
||||
|
||||
"fast-uri": ["fast-uri@3.1.0", "", {}, "sha512-iPeeDKJSWf4IEOasVVrknXpaBV0IApz/gp7S2bb7Z4Lljbl2MGJRqInZiUrQwV16cpzw/D3S5j5Julj/gT52AA=="],
|
||||
|
||||
"finalhandler": ["finalhandler@2.1.1", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA=="],
|
||||
|
||||
"form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="],
|
||||
|
||||
"form-data-encoder": ["form-data-encoder@1.7.2", "", {}, "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A=="],
|
||||
|
||||
"formdata-node": ["formdata-node@4.4.1", "", { "dependencies": { "node-domexception": "1.0.0", "web-streams-polyfill": "4.0.0-beta.3" } }, "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ=="],
|
||||
|
||||
"forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
|
||||
|
||||
"fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
|
||||
@@ -164,16 +130,12 @@
|
||||
|
||||
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
|
||||
|
||||
"has-tostringtag": ["has-tostringtag@1.0.2", "", { "dependencies": { "has-symbols": "^1.0.3" } }, "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw=="],
|
||||
|
||||
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"hono": ["hono@4.12.9", "", {}, "sha512-wy3T8Zm2bsEvxKZM5w21VdHDDcwVS1yUFFY6i8UobSsKfFceT7TOwhbhfKsDyx7tYQlmRM5FLpIuYvNFyjctiA=="],
|
||||
|
||||
"http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="],
|
||||
|
||||
"humanize-ms": ["humanize-ms@1.2.1", "", { "dependencies": { "ms": "^2.0.0" } }, "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.7.2", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw=="],
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
@@ -208,10 +170,6 @@
|
||||
|
||||
"negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
|
||||
|
||||
"node-domexception": ["node-domexception@1.0.0", "", {}, "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ=="],
|
||||
|
||||
"node-fetch": ["node-fetch@2.7.0", "", { "dependencies": { "whatwg-url": "^5.0.0" }, "peerDependencies": { "encoding": "^0.1.0" }, "optionalPeers": ["encoding"] }, "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A=="],
|
||||
|
||||
"object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="],
|
||||
|
||||
"object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
|
||||
@@ -262,30 +220,24 @@
|
||||
|
||||
"side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
|
||||
|
||||
"standardwebhooks": ["standardwebhooks@1.1.1", "", { "dependencies": { "@stablelib/base64": "^1.0.0", "fast-sha256": "^1.3.0" } }, "sha512-bCbX9ZEyFkWPsRz7Bl3NuQUJohmwGSev/yhr7vhaGPlc4AfIrspIRa6cPTBuI1ItmrTDJ4d/S2hCsfe4+vQGnQ=="],
|
||||
|
||||
"statuses": ["statuses@2.0.2", "", {}, "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw=="],
|
||||
|
||||
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
|
||||
|
||||
"tr46": ["tr46@0.0.3", "", {}, "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="],
|
||||
|
||||
"ts-algebra": ["ts-algebra@2.0.0", "", {}, "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw=="],
|
||||
|
||||
"type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici-types": ["undici-types@5.26.5", "", {}, "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA=="],
|
||||
"undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
|
||||
|
||||
"vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
|
||||
|
||||
"web-streams-polyfill": ["web-streams-polyfill@4.0.0-beta.3", "", {}, "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug=="],
|
||||
|
||||
"webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="],
|
||||
|
||||
"whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="],
|
||||
|
||||
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
|
||||
|
||||
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
|
||||
@@ -295,19 +247,5 @@
|
||||
"zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="],
|
||||
|
||||
"zod-to-json-schema": ["zod-to-json-schema@3.25.2", "", { "peerDependencies": { "zod": "^3.25.28 || ^4" } }, "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA=="],
|
||||
|
||||
"@anthropic-ai/claude-agent-sdk/@anthropic-ai/sdk": ["@anthropic-ai/sdk@0.74.0", "", { "dependencies": { "json-schema-to-ts": "^3.1.1" }, "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["zod"], "bin": { "anthropic-ai-sdk": "bin/cli" } }, "sha512-srbJV7JKsc5cQ6eVuFzjZO7UR3xEPJqPamHFIe29bs38Ij2IripoAhC0S5NslNbaFUYqBKypmmpzMTpqfHEUDw=="],
|
||||
|
||||
"@types/node-fetch/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
|
||||
|
||||
"bun-types/@types/node": ["@types/node@25.5.0", "", { "dependencies": { "undici-types": "~7.18.0" } }, "sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw=="],
|
||||
|
||||
"form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="],
|
||||
|
||||
"@types/node-fetch/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"bun-types/@types/node/undici-types": ["undici-types@7.18.2", "", {}, "sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w=="],
|
||||
|
||||
"form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +108,10 @@ async function main() {
|
||||
"--record",
|
||||
"append a compact summary line to ai_evals/history/<mode>.jsonl",
|
||||
)
|
||||
.option(
|
||||
"--reasoning <effort>",
|
||||
"reasoning effort for frontend modes (e.g. off, low, medium, high, max); default: the product's",
|
||||
)
|
||||
.option(
|
||||
"--backend-validation <mode>",
|
||||
`backend smoke validation (${BACKEND_VALIDATION_MODES.join(", ")})`,
|
||||
@@ -126,6 +130,7 @@ async function main() {
|
||||
executionOnly?: boolean;
|
||||
record?: boolean;
|
||||
backendValidation?: string;
|
||||
reasoning?: string;
|
||||
},
|
||||
) => {
|
||||
await handleRun({
|
||||
@@ -140,6 +145,7 @@ async function main() {
|
||||
executionOnly: options.executionOnly ?? false,
|
||||
record: options.record ?? false,
|
||||
backendValidation: options.backendValidation,
|
||||
reasoning: options.reasoning,
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -190,6 +196,7 @@ async function handleRun(input: {
|
||||
executionOnly: boolean;
|
||||
record: boolean;
|
||||
backendValidation?: string;
|
||||
reasoning?: string;
|
||||
}) {
|
||||
if (input.record && input.caseIds.length > 0) {
|
||||
throw new Error(
|
||||
@@ -217,6 +224,13 @@ async function handleRun(input: {
|
||||
"--backend-validation currently supports only flow and script modes",
|
||||
);
|
||||
}
|
||||
if (input.reasoning) {
|
||||
if (input.mode === "cli") {
|
||||
throw new Error("--reasoning only applies to frontend modes");
|
||||
}
|
||||
// The frontend runtime runs in a child process, which inherits it.
|
||||
process.env.WMILL_AI_EVAL_REASONING = input.reasoning;
|
||||
}
|
||||
if (input.mode !== "cli") {
|
||||
await assertWindmillBackendReachable(resolveWindmillBackendSettings());
|
||||
}
|
||||
@@ -257,6 +271,10 @@ async function handleRun(input: {
|
||||
backendValidation,
|
||||
});
|
||||
|
||||
if (input.reasoning && result.runModel) {
|
||||
result.runModel = `${result.runModel}@${input.reasoning}`;
|
||||
}
|
||||
|
||||
const resolvedOutputPath =
|
||||
models.length === 1
|
||||
? resolveRunOutputPath(input.mode, input.outputPath)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import Anthropic from "@anthropic-ai/sdk";
|
||||
import type { EvalMode, JudgeResult } from "./types";
|
||||
|
||||
export const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-6";
|
||||
export const DEFAULT_JUDGE_MODEL = "claude-sonnet-5-5";
|
||||
|
||||
const JUDGE_TOOL_NAME = "submit_judgement";
|
||||
|
||||
@@ -29,6 +29,7 @@ export async function judgeOutput(input: {
|
||||
|
||||
const system = [
|
||||
"You evaluate benchmark outputs for Windmill AI generation.",
|
||||
`Always answer by calling the ${JUDGE_TOOL_NAME} tool.`,
|
||||
"Deterministic checks already run separately. Focus on whether the final output satisfies the user request.",
|
||||
"If expected state is provided, treat it as a valid example and reward semantically equivalent outputs.",
|
||||
"If a checklist is provided, treat it as the explicit acceptance criteria for this case.",
|
||||
@@ -69,8 +70,8 @@ export async function judgeOutput(input: {
|
||||
try {
|
||||
const response = await client.messages.create({
|
||||
model,
|
||||
max_tokens: 1024,
|
||||
temperature: 0,
|
||||
// The judge thinks by default, and thinking shares this budget with the verdict.
|
||||
max_tokens: 16000,
|
||||
system,
|
||||
messages: [{ role: "user", content: user }],
|
||||
tools: [
|
||||
@@ -93,11 +94,8 @@ export async function judgeOutput(input: {
|
||||
},
|
||||
},
|
||||
],
|
||||
tool_choice: {
|
||||
type: "tool",
|
||||
name: JUDGE_TOOL_NAME,
|
||||
disable_parallel_tool_use: true,
|
||||
},
|
||||
// Current models refuse a forced tool_choice ("tool"/"any").
|
||||
tool_choice: { type: "auto", disable_parallel_tool_use: true },
|
||||
});
|
||||
|
||||
const toolUseBlock = response.content.find(
|
||||
|
||||
@@ -78,6 +78,32 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "opus",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "sonnet-5.5",
|
||||
label: "Claude Sonnet 5.5",
|
||||
aliases: ["sonnet-5.5", "claude-sonnet-5.5", "claude-sonnet-5-5"],
|
||||
frontend: {
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-5-5",
|
||||
},
|
||||
cli: {
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-5-5",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "opus-5.5",
|
||||
label: "Claude Opus 5.5",
|
||||
aliases: ["opus-5.5", "claude-opus-5.5", "claude-opus-5-5"],
|
||||
frontend: {
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-5-5",
|
||||
},
|
||||
cli: {
|
||||
provider: "anthropic",
|
||||
model: "claude-opus-5-5",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "4o",
|
||||
label: "GPT-4o",
|
||||
@@ -96,6 +122,51 @@ export const EVAL_MODELS: EvalModelSpec[] = [
|
||||
model: "gpt-5.5",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gpt-5.6-sol",
|
||||
label: "GPT-5.6 Sol",
|
||||
aliases: ["gpt-5.6-sol"],
|
||||
frontend: {
|
||||
provider: "openai",
|
||||
model: "gpt-5.6-sol",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gpt-6-astra",
|
||||
label: "GPT-6 Astra",
|
||||
aliases: ["gpt-6-astra", "gpt-6"],
|
||||
frontend: {
|
||||
provider: "openai",
|
||||
model: "gpt-6-astra",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gpt-6-sol",
|
||||
label: "GPT-6 Sol",
|
||||
aliases: ["gpt-6-sol"],
|
||||
frontend: {
|
||||
provider: "openai",
|
||||
model: "gpt-6-sol",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gpt-6-luna",
|
||||
label: "GPT-6 Luna",
|
||||
aliases: ["gpt-6-luna"],
|
||||
frontend: {
|
||||
provider: "openai",
|
||||
model: "gpt-6-luna",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gemini-3.8-flash",
|
||||
label: "Gemini 3.8 Flash",
|
||||
aliases: ["gemini-3.8-flash"],
|
||||
frontend: {
|
||||
provider: "googleai",
|
||||
model: "gemini-3.8-flash",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "gemini-3-flash-preview",
|
||||
label: "Gemini 3 Flash Preview",
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
"test:frontend-graph": "cd ../frontend && node_modules/.bin/vitest run --project server --config ../ai_evals/adapters/frontend/vitest.unit.config.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.2.25",
|
||||
"@anthropic-ai/sdk": "^0.39.0",
|
||||
"@anthropic-ai/claude-agent-sdk": "^0.3.284",
|
||||
"@anthropic-ai/sdk": "^0.129.0",
|
||||
"commander": "^14.0.3",
|
||||
"openai": "^6.9.1",
|
||||
"yaml": "^2.8.3"
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH RECURSIVE descendants AS (\n SELECT id FROM v2_job WHERE parent_job = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id FROM v2_job j JOIN descendants d ON j.parent_job = d.id\n WHERE j.workspace_id = $2\n ) SELECT d.id AS \"id!\" FROM descendants d JOIN v2_job_queue q ON q.id = d.id",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id!",
|
||||
"type_info": "Uuid"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "0c2bf6925de6dd4d9f8d47d7fdd9e7801129570106b5acd49c0f708398606771"
|
||||
}
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET started_at = now() WHERE id = $1 RETURNING started_at",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "started_at",
|
||||
"type_info": "Timestamptz"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "451f303f4a24d848ba4ccdc2441ca7cb85bdfc66110b6534260f8642f91f006a"
|
||||
}
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT c.id IS NOT NULL AS \"completed!\",\n c.result AS \"result: sqlx::types::Json<Box<RawValue>>\",\n c.status = 'success' AS \"success\",\n EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1) AS \"queued!\"\n FROM (SELECT 1) one\n LEFT JOIN v2_job_completed c ON c.id = $1 AND c.workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "completed!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "result: sqlx::types::Json<Box<RawValue>>",
|
||||
"type_info": "Jsonb"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "success",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "queued!",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Uuid",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
true,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "52f9582423689f098a337f88341239a136ee5132bc97f4b06cc1ee5beaa621e5"
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT id, status = 'success' AS \"success!\", result AS \"result: Json<Box<RawValue>>\"\n FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Uuid"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "success!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "result: Json<Box<RawValue>>",
|
||||
"type_info": "Jsonb"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"UuidArray"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
null,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "789c3f6d29f46fcb17a22fe97405a1754ccb8ddeb08c23795dfe14e9f332e9dc"
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE v2_job_queue SET worker = $1, running = true, started_at = now()\n WHERE id = $2 AND tag = ANY($3)",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Uuid",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "f5c87b6fc1decd6dc53590fe4d16f59430530b8b7428e42d1777927e63395ff8"
|
||||
}
|
||||
Generated
+10
@@ -14876,6 +14876,7 @@ dependencies = [
|
||||
"windmill-worker-volumes",
|
||||
"windows-service",
|
||||
"windows-sys 0.52.0",
|
||||
"winresource",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -17382,6 +17383,15 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winresource"
|
||||
version = "0.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0986a8b1d586b7d3e4fe3d9ea39fb451ae22869dcea4aa109d287a374d866087"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winsafe"
|
||||
version = "0.0.19"
|
||||
|
||||
@@ -329,6 +329,9 @@ strum.workspace = true
|
||||
windows-service = "0.7"
|
||||
windows-sys = { version = "0.52", features = ["Win32_System_Services", "Win32_System_Console", "Win32_Foundation"] }
|
||||
|
||||
[target.'cfg(windows)'.build-dependencies]
|
||||
winresource = { version = "0.1.31", default-features = false }
|
||||
|
||||
[target.'cfg(not(target_env = "msvc"))'.dependencies]
|
||||
tikv-jemallocator = { optional = true, workspace = true }
|
||||
tikv-jemalloc-sys = { optional = true, workspace = true }
|
||||
|
||||
@@ -2,4 +2,19 @@
|
||||
fn main() {
|
||||
// trigger recompilation when a new migration is added
|
||||
println!("cargo:rerun-if-changed=migrations");
|
||||
|
||||
// Embed a VERSIONINFO resource so the exe's file properties carry the release
|
||||
// version (from Cargo.toml), which Windows worker update scripts read.
|
||||
// `cfg(windows)` is the host, matching the Windows-only build-dependency, so
|
||||
// cross builds to Windows get no version resource.
|
||||
#[cfg(windows)]
|
||||
if std::env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("windows") {
|
||||
winresource::WindowsResource::new()
|
||||
.set("ProductName", "Windmill")
|
||||
.set("FileDescription", "Windmill")
|
||||
.set("CompanyName", "Windmill Labs, Inc.")
|
||||
.set("LegalCopyright", "Copyright Windmill Labs, Inc.")
|
||||
.compile()
|
||||
.expect("failed to embed the Windows version resource");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
use axum::{
|
||||
extract::State,
|
||||
routing::{get, post},
|
||||
Json, Router,
|
||||
};
|
||||
use serde_json::{json, Value};
|
||||
use sqlx::{Pool, Postgres};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use tokio::sync::Notify;
|
||||
use windmill_common::{flows::FlowValue, jobs::JobPayload};
|
||||
use windmill_test_utils::{
|
||||
completed_job, in_test_worker, listen_for_completed_jobs, ApiServer, RunJob, StreamFind,
|
||||
};
|
||||
|
||||
async fn model(Json(body): Json<Value>) -> ([(&'static str, &'static str); 1], String) {
|
||||
let finished = body["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|m| m["role"] == "tool");
|
||||
let delta = if finished {
|
||||
json!({"role":"assistant", "content":"done"})
|
||||
} else {
|
||||
json!({"role":"assistant", "tool_calls": (0..3).map(|i| json!({
|
||||
"index":i, "id":format!("call_{i}"), "type":"function",
|
||||
"function":{"name":format!("tool_{i}"), "arguments":"{}"}
|
||||
})).collect::<Vec<_>>()})
|
||||
};
|
||||
let event = json!({"choices":[{"index":0, "delta":delta, "finish_reason":null}]});
|
||||
let end = json!({"choices":[{"index":0, "delta":{}, "finish_reason":if finished {"stop"} else {"tool_calls"}}]});
|
||||
(
|
||||
[("content-type", "text/event-stream")],
|
||||
format!("data: {event}\n\ndata: {end}\n\ndata: [DONE]\n\n"),
|
||||
)
|
||||
}
|
||||
|
||||
async fn run_batch(
|
||||
db: Pool<Postgres>,
|
||||
parallel: bool,
|
||||
limited: bool,
|
||||
unsupported: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
std::env::set_var("ALLOW_PRIVATE_AI_BASE_URLS", "true");
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
|
||||
let base = format!("http://{}", listener.local_addr()?);
|
||||
let gate = Arc::new(Notify::new());
|
||||
let router = Router::new()
|
||||
.route("/v1/chat/completions", post(model))
|
||||
.route(
|
||||
"/first",
|
||||
get(|State(gate): State<Arc<Notify>>| async move {
|
||||
gate.notified().await;
|
||||
"ok"
|
||||
}),
|
||||
)
|
||||
.route(
|
||||
"/second",
|
||||
get(|State(gate): State<Arc<Notify>>| async move {
|
||||
gate.notify_one();
|
||||
"ok"
|
||||
}),
|
||||
)
|
||||
.with_state(gate);
|
||||
let stub = tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
});
|
||||
let tools: Vec<Value> = (0..3).map(|i| {
|
||||
let wait = if parallel && !limited && i < 2 {
|
||||
format!("await fetch('{base}/{}');", if i == 0 {"first"} else {"second"})
|
||||
} else { String::new() };
|
||||
let finish = if i == 2 { "throw new Error('expected tool failure');".to_string() }
|
||||
else { format!("return {i};") };
|
||||
json!({"id":format!("t{i}"),"summary":format!("tool_{i}"),"value":{
|
||||
"type":"rawscript", "language":"bun", "input_transforms":{},
|
||||
"tag": if unsupported && i != 1 { "unserved-tool-tag" } else { "bun" },
|
||||
"concurrent_limit": if limited { Some(1) } else { None },
|
||||
"custom_concurrency_key": if limited { Some("agent-tool-test") } else { None },
|
||||
"content":format!("export async function main() {{ {wait} await Bun.sleep({}); {finish} }}", if i == 0 {200} else {0})
|
||||
}})
|
||||
}).collect();
|
||||
let flow: FlowValue = serde_json::from_value(
|
||||
json!({"modules":[{"id":"agent","timeout":if unsupported {Some(3)} else {None},"value":{
|
||||
"type":"aiagent", "tools":tools, "input_transforms":{
|
||||
"provider":{"type":"static","value":{"kind":"customai","model":"queue-test","resource":{"base_url":format!("{base}/v1")}}},
|
||||
"user_message":{"type":"static","value":"run the tools"},
|
||||
"max_iterations":{"type":"static","value":3}
|
||||
}
|
||||
}}]}),
|
||||
)?;
|
||||
let id = RunJob::from(JobPayload::RawFlow {
|
||||
value: flow,
|
||||
path: Some("u/test/agent_queue".into()),
|
||||
restarted_from: None,
|
||||
})
|
||||
.push(&db)
|
||||
.await;
|
||||
let notifications = listen_for_completed_jobs(&db).await;
|
||||
let wait = async {
|
||||
if parallel {
|
||||
in_test_worker(&db, notifications.find(&id), server.addr.port()).await;
|
||||
} else {
|
||||
notifications.find(&id).await;
|
||||
}
|
||||
};
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(45),
|
||||
in_test_worker(&db, wait, server.addr.port()),
|
||||
)
|
||||
.await?;
|
||||
let result = completed_job(id, &db).await;
|
||||
if unsupported {
|
||||
assert!(
|
||||
!result.success,
|
||||
"agent must time out waiting for unsupported tools"
|
||||
);
|
||||
let children: Vec<(String, bool, bool)> = sqlx::query_as(
|
||||
"SELECT j.runnable_path, c.status = 'success', c.started_at IS NULL
|
||||
FROM v2_job j JOIN v2_job_completed c USING(id)
|
||||
WHERE j.parent_job IN (SELECT id FROM v2_job WHERE parent_job = $1)
|
||||
ORDER BY j.runnable_path",
|
||||
)
|
||||
.bind(id)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
assert_eq!(children.len(), 3);
|
||||
assert_eq!(
|
||||
children.iter().map(|c| (c.1, c.2)).collect::<Vec<_>>(),
|
||||
vec![(false, true), (true, false), (false, true)],
|
||||
"neither the first-tool reservation nor fallback may execute unsupported tags"
|
||||
);
|
||||
stub.abort();
|
||||
server.close().await?;
|
||||
return Ok(());
|
||||
}
|
||||
assert!(result.success, "{:?}", result.result);
|
||||
let result = result.json_result().expect("agent result");
|
||||
let messages: Vec<_> = result["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|m| m["role"] == "tool")
|
||||
.collect();
|
||||
assert_eq!(messages.len(), 3);
|
||||
for (index, message) in messages.iter().enumerate() {
|
||||
assert_eq!(message["tool_call_id"], format!("call_{index}"));
|
||||
}
|
||||
assert_eq!(messages[0]["content"], "0");
|
||||
assert_eq!(messages[1]["content"], "1");
|
||||
assert!(messages[2]["content"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("expected tool failure"));
|
||||
let (parent_id, parent_worker): (uuid::Uuid, String) = sqlx::query_as(
|
||||
"SELECT j.id, c.worker FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1"
|
||||
).bind(id).fetch_one(&db).await?;
|
||||
let children: Vec<(String, String, bool)> = sqlx::query_as(
|
||||
"SELECT j.runnable_path, c.worker, c.status = 'success' FROM v2_job j JOIN v2_job_completed c USING(id) WHERE j.parent_job = $1 ORDER BY j.runnable_path"
|
||||
).bind(parent_id).fetch_all(&db).await?;
|
||||
assert_eq!(children.len(), 3);
|
||||
assert_eq!(
|
||||
children[0].1, parent_worker,
|
||||
"first tool must stay on the parent worker"
|
||||
);
|
||||
assert_eq!(
|
||||
children.iter().map(|c| c.2).collect::<Vec<_>>(),
|
||||
vec![true, true, false]
|
||||
);
|
||||
if limited {
|
||||
let overlapping: i64 = sqlx::query_scalar(
|
||||
"SELECT count(*) FROM v2_job j1 JOIN v2_job_completed c1 ON c1.id = j1.id
|
||||
JOIN v2_job j2 ON j2.parent_job = j1.parent_job AND j2.id > j1.id
|
||||
JOIN v2_job_completed c2 ON c2.id = j2.id
|
||||
WHERE j1.parent_job = $1 AND c1.started_at < c2.completed_at AND c2.started_at < c1.completed_at"
|
||||
).bind(parent_id).fetch_one(&db).await?;
|
||||
assert_eq!(
|
||||
overlapping, 0,
|
||||
"the reserved first job must count toward the shared limit"
|
||||
);
|
||||
} else if parallel {
|
||||
assert_ne!(
|
||||
children[1].1, parent_worker,
|
||||
"second tool must unblock the first from another worker"
|
||||
);
|
||||
} else {
|
||||
assert!(children.iter().all(|child| child.1 == parent_worker));
|
||||
}
|
||||
stub.abort();
|
||||
server.close().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
#[serial_test::serial]
|
||||
async fn parent_drains_tools_without_another_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
run_batch(db, false, false, false).await
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
#[serial_test::serial]
|
||||
async fn parent_keeps_first_tool_while_another_worker_runs_siblings(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
run_batch(db, true, false, false).await
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
#[serial_test::serial]
|
||||
async fn reserved_tool_obeys_shared_concurrency_limit(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
run_batch(db, true, true, false).await
|
||||
}
|
||||
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
#[serial_test::serial]
|
||||
async fn parent_leaves_unsupported_first_and_remaining_tools_queued(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
run_batch(db, false, false, true).await
|
||||
}
|
||||
@@ -465,6 +465,37 @@ def main(x: int):
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_python_execd_async() {
|
||||
let script = r#"
|
||||
import asyncio
|
||||
|
||||
_import_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(_import_loop)
|
||||
|
||||
async def preprocessor(x: int):
|
||||
return {"x": x * 2}
|
||||
|
||||
async def main(x: int):
|
||||
return {"x": x + 100, "same_loop": asyncio.get_running_loop() is _import_loop}
|
||||
"#;
|
||||
let results = run_py_raw_protocol_test(
|
||||
&[("f/test/async", script)],
|
||||
vec![
|
||||
ProtocolCmd::ExecdPreprocess { args: serde_json::json!({"x": 5}) },
|
||||
ProtocolCmd::Execd { args: serde_json::json!({"x": 7}) },
|
||||
],
|
||||
);
|
||||
assert_eq!(
|
||||
results,
|
||||
vec![
|
||||
DedicatedWorkerResult::PreprocessedArgs(serde_json::json!({"x": 10})),
|
||||
DedicatedWorkerResult::Success(serde_json::json!({"x": 110, "same_loop": true})),
|
||||
DedicatedWorkerResult::Success(serde_json::json!({"x": 107, "same_loop": true})),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== Argument Transformation Tests ====================
|
||||
|
||||
#[test]
|
||||
@@ -760,6 +791,49 @@ def main():
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "python")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_python_async_main(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let content = r#"
|
||||
import asyncio
|
||||
|
||||
async def main(x: int):
|
||||
await asyncio.sleep(0)
|
||||
return x + 1
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
language: ScriptLang::Python3,
|
||||
lock: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
modules: None,
|
||||
tag: None,
|
||||
});
|
||||
|
||||
let result = RunJob::from(job)
|
||||
.arg("x", serde_json::json!(41))
|
||||
.run_until_complete(&db, false, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(42));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "python")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_python_result_preserves_infinity_in_string(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
|
||||
@@ -14,6 +14,130 @@ use std::time::{Duration, Instant};
|
||||
/// its `thinking` param, Gemini to a zero budget or the model's floor).
|
||||
pub(crate) const REASONING_OFF_SENTINEL: &str = "none";
|
||||
|
||||
/// The effort to send for a model, dropping the off sentinel on a model that rejects every
|
||||
/// disable: the model then reasons at its default instead of failing the request. The UI
|
||||
/// never offers off on these, so this guards an agent step saved before it stopped, or an
|
||||
/// effort passed in as a flow input.
|
||||
pub fn effective_reasoning_effort<'a>(model: &str, effort: Option<&'a str>) -> Option<&'a str> {
|
||||
match effort {
|
||||
Some(effort)
|
||||
if effort == REASONING_OFF_SENTINEL
|
||||
&& reasoning_rule(model).is_some_and(|rule| !rule.can_disable) =>
|
||||
{
|
||||
None
|
||||
}
|
||||
effort => effort,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a request with function tools must send the off sentinel on Chat Completions:
|
||||
/// the model refuses tools there while it reasons, and does accept being turned off.
|
||||
pub(crate) fn completions_tools_need_reasoning_off(model: &str) -> bool {
|
||||
reasoning_rule(model).is_some_and(|rule| rule.completions_tools_need_off && rule.can_disable)
|
||||
}
|
||||
|
||||
/// What the backend needs to know about a model's reasoning: the rows of `REASONING_RULES`
|
||||
/// in the frontend's `reasoningRegistry.ts`, cut down to the two facts the wire needs.
|
||||
/// `reasoningParity.json` next to that file is checked by both sides' tests.
|
||||
struct ReasoningRule {
|
||||
matches: fn(&str) -> bool,
|
||||
/// False when the provider rejects every disable, so the off sentinel must not be sent.
|
||||
can_disable: bool,
|
||||
/// Chat Completions refuses function tools while the model reasons, even with the
|
||||
/// effort omitted (live-verified); the Responses API has no such limit.
|
||||
completions_tools_need_off: bool,
|
||||
}
|
||||
|
||||
/// Matched in order against the lowercased model id, first match wins. A model no row
|
||||
/// matches keeps whatever effort it was given.
|
||||
const REASONING_RULES: &[ReasoningRule] = &[
|
||||
// Live-verified: Fable, Mythos and the 5.x point releases reject `thinking: disabled`.
|
||||
ReasoningRule {
|
||||
matches: |m| m.contains("claude-fable") || m.contains("claude-mythos"),
|
||||
can_disable: false,
|
||||
completions_tools_need_off: false,
|
||||
},
|
||||
ReasoningRule {
|
||||
matches: is_claude_5_point_release,
|
||||
can_disable: false,
|
||||
completions_tools_need_off: false,
|
||||
},
|
||||
// Live-verified: astra takes low..max only, where sol and luna also take `none`.
|
||||
ReasoningRule {
|
||||
matches: |m| base_id(m).starts_with("gpt-6-astra"),
|
||||
can_disable: false,
|
||||
completions_tools_need_off: true,
|
||||
},
|
||||
ReasoningRule {
|
||||
matches: |m| gpt_version(m).is_some_and(|(major, _)| major >= 6),
|
||||
can_disable: true,
|
||||
completions_tools_need_off: true,
|
||||
},
|
||||
ReasoningRule {
|
||||
matches: |m| matches!(gpt_version(m), Some((5, Some(minor))) if minor >= 5),
|
||||
can_disable: true,
|
||||
completions_tools_need_off: true,
|
||||
},
|
||||
ReasoningRule {
|
||||
matches: |m| matches!(gpt_version(m), Some((5, Some(_)))),
|
||||
can_disable: true,
|
||||
completions_tools_need_off: false,
|
||||
},
|
||||
// gpt-5 and the o-series reject `none`.
|
||||
ReasoningRule {
|
||||
matches: |m| matches!(gpt_version(m), Some((5, None))),
|
||||
can_disable: false,
|
||||
completions_tools_need_off: false,
|
||||
},
|
||||
ReasoningRule {
|
||||
matches: |m| {
|
||||
let base = base_id(m);
|
||||
base.starts_with('o') && base[1..].starts_with(|c: char| c.is_ascii_digit())
|
||||
},
|
||||
can_disable: false,
|
||||
completions_tools_need_off: false,
|
||||
},
|
||||
];
|
||||
|
||||
fn reasoning_rule(model: &str) -> Option<&'static ReasoningRule> {
|
||||
let model = model.to_lowercase();
|
||||
REASONING_RULES.iter().find(|rule| (rule.matches)(&model))
|
||||
}
|
||||
|
||||
/// The id after a gateway's `vendor/` and before a `:variant`.
|
||||
fn base_id(model: &str) -> &str {
|
||||
let last = model.rsplit('/').next().unwrap_or(model);
|
||||
last.split(':').next().unwrap_or(last)
|
||||
}
|
||||
|
||||
/// `(major, minor)` of a `gpt-` id. The major is one digit then a separator or the end,
|
||||
/// since Azure names gpt-3.5 `gpt-35-turbo`.
|
||||
fn gpt_version(model: &str) -> Option<(u32, Option<u32>)> {
|
||||
let rest = base_id(model).strip_prefix("gpt-")?;
|
||||
let mut chars = rest.chars();
|
||||
let major = chars.next()?.to_digit(10)?;
|
||||
match chars.next() {
|
||||
None | Some('-') => Some((major, None)),
|
||||
Some('.') => {
|
||||
let minor: String = chars.take_while(char::is_ascii_digit).collect();
|
||||
Some((major, minor.parse().ok()))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sonnet or Opus 5.x with x >= 1. The version match stops at one digit so a dated id
|
||||
/// (`claude-sonnet-5-20260101`) stays Sonnet 5.
|
||||
fn is_claude_5_point_release(model: &str) -> bool {
|
||||
let model = model.replace('.', "-");
|
||||
["claude-opus-5-", "claude-sonnet-5-"].iter().any(|prefix| {
|
||||
model.split(prefix).skip(1).any(|rest| {
|
||||
let mut chars = rest.chars();
|
||||
matches!(chars.next(), Some('1'..='9')) && !matches!(chars.next(), Some('0'..='9'))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether a Claude model removed the sampling params (`temperature`, `top_p`,
|
||||
/// `top_k`). On these, any value is a hard 400 — `temperature is deprecated for
|
||||
/// this model` — whatever the thinking mode, so the param has to be dropped on
|
||||
@@ -119,6 +243,46 @@ pub fn remember_chat_completions_only(base_url: &str, model: &str) {
|
||||
CHAT_COMPLETIONS_ONLY.insert((base_url.to_string(), model.to_string()), Instant::now());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod reasoning_rule_tests {
|
||||
use super::*;
|
||||
|
||||
/// The frontend registry's test reads the same file. These rules see the model id
|
||||
/// alone, so it holds only rows whose answer doesn't depend on the provider.
|
||||
#[test]
|
||||
fn agrees_with_the_frontend_registry() {
|
||||
let rows: Vec<serde_json::Value> = serde_json::from_str(include_str!(
|
||||
"../../../../frontend/src/lib/components/copilot/reasoningParity.json"
|
||||
))
|
||||
.unwrap();
|
||||
for row in rows {
|
||||
let model = row["model"].as_str().unwrap();
|
||||
let can_disable = row["canDisable"].as_bool().unwrap();
|
||||
let tools_need_off = row["completionsToolsNeedOff"].as_bool().unwrap();
|
||||
let sent = effective_reasoning_effort(model, Some(REASONING_OFF_SENTINEL));
|
||||
assert_eq!(sent.is_some(), can_disable, "{model}");
|
||||
assert_eq!(
|
||||
effective_reasoning_effort(model, Some("low")),
|
||||
Some("low"),
|
||||
"{model}"
|
||||
);
|
||||
assert_eq!(
|
||||
completions_tools_need_reasoning_off(model),
|
||||
tools_need_off && can_disable,
|
||||
"{model}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_ids_the_frontend_resolves_first() {
|
||||
// Azure's gpt-3.5 is not major 35, and a gateway prefix is not part of the id.
|
||||
assert_eq!(gpt_version("gpt-35-turbo"), None);
|
||||
assert!(completions_tools_need_reasoning_off("openai/gpt-6-sol"));
|
||||
assert_eq!(effective_reasoning_effort("openai/o3", Some("none")), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod chat_completions_only_tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use super::{completions_tools_need_reasoning_off, REASONING_OFF_SENTINEL};
|
||||
use crate::{
|
||||
ai_providers::AIProvider,
|
||||
image_handler::prepare_messages_for_api,
|
||||
@@ -143,6 +144,15 @@ impl OtherQueryBuilder {
|
||||
|
||||
let (reasoning_effort, thinking, temperature) =
|
||||
provider_reasoning_fields(&self.provider_kind, args.reasoning_effort, args.temperature);
|
||||
// With tools, gpt-5.5+ only run here with reasoning off: turn it off where the model
|
||||
// can, rather than failing every turn.
|
||||
let reasoning_effort = if args.tools.is_some_and(|tools| !tools.is_empty())
|
||||
&& completions_tools_need_reasoning_off(args.model)
|
||||
{
|
||||
Some(REASONING_OFF_SENTINEL)
|
||||
} else {
|
||||
reasoning_effort
|
||||
};
|
||||
|
||||
// Build request with stream_options for usage tracking
|
||||
let request_with_usage = OpenAICompletionRequest {
|
||||
|
||||
@@ -294,7 +294,10 @@ impl ProviderWithResource {
|
||||
/// The reasoning effort to thread to the provider, treating an empty string
|
||||
/// (e.g. a cleared flow input) as unset.
|
||||
pub fn get_reasoning_effort(&self) -> Option<&str> {
|
||||
self.reasoning_effort.as_deref().filter(|s| !s.is_empty())
|
||||
crate::providers::effective_reasoning_effort(
|
||||
&self.model,
|
||||
self.reasoning_effort.as_deref().filter(|s| !s.is_empty()),
|
||||
)
|
||||
}
|
||||
|
||||
pub async fn get_base_url(&self, db: &DB) -> Result<String, Error> {
|
||||
|
||||
@@ -864,6 +864,24 @@ pub fn make_pull_query(tags: &[String]) -> String {
|
||||
query
|
||||
}
|
||||
|
||||
/// Claim a parent's tool jobs matching its worker tags. The caller must supply only child IDs
|
||||
/// it owns; this query is an internal scheduling primitive and does not authorize job access.
|
||||
pub fn make_tool_job_pull_query(job_ids: &[uuid::Uuid], tags: &[String]) -> String {
|
||||
// pull() binds only the worker name. These literals come from typed UUIDs, never input SQL.
|
||||
let ids = job_ids.iter().map(|id| format!("'{id}'::uuid")).join(", ");
|
||||
let tags = tags
|
||||
.iter()
|
||||
.map(|tag| format!("'{}'", tag.replace('\'', "''")))
|
||||
.join(", ");
|
||||
format_pull_query(format!(
|
||||
"SELECT id FROM v2_job_queue
|
||||
WHERE running = false AND id = ANY(ARRAY[{ids}]::uuid[]) AND scheduled_for <= now()
|
||||
AND tag = ANY(ARRAY[{tags}]::text[])
|
||||
ORDER BY priority DESC NULLS LAST, scheduled_for
|
||||
FOR UPDATE SKIP LOCKED LIMIT 1"
|
||||
))
|
||||
}
|
||||
|
||||
// Variant of `make_pull_query` that additionally excludes jobs whose workspace_id is in the
|
||||
// overloaded-list bind parameter ($2::text[]). Built as a separate string (rather than reusing
|
||||
// `make_pull_query` with an always-bound array) so the planner can keep using the same indexes
|
||||
|
||||
@@ -184,7 +184,7 @@ pub enum ObjectType {
|
||||
DatatableMigration,
|
||||
}
|
||||
|
||||
pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo-windmill";
|
||||
pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28994/sync-script-to-git-repo-windmill";
|
||||
|
||||
/// Hub script that applies a repository's state back into a workspace
|
||||
/// (the repo → Windmill / "pull" direction). Same script the UI runs from
|
||||
@@ -192,7 +192,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28971/sync-script-to-git-repo
|
||||
/// ignores the slug, so the slug is kept free of characters that would be
|
||||
/// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened
|
||||
/// reverse proxies reject as double-encoding when the client re-encodes it).
|
||||
pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28970/git-sync-init-repository-windmill";
|
||||
pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28993/git-sync-init-repository-windmill";
|
||||
|
||||
/// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a
|
||||
/// fork of another workspace.
|
||||
|
||||
@@ -4909,6 +4909,45 @@ pub fn has_active_concurrency_limit(concurrent_limit: Option<i32>) -> bool {
|
||||
concurrent_limit.is_some_and(|n| n > 0)
|
||||
}
|
||||
|
||||
/// Admit a job already owned by a worker without releasing its queue reservation.
|
||||
/// The caller must maintain its heartbeat while waiting and complete it on failure.
|
||||
pub async fn try_admit_owned_job(db: &DB, job: &MiniPulledJob) -> error::Result<bool> {
|
||||
#[cfg(all(feature = "private", feature = "enterprise"))]
|
||||
{
|
||||
let settings = windmill_common::runnable_settings::prefetch_cached_from_handle(
|
||||
job.runnable_settings_handle,
|
||||
db,
|
||||
)
|
||||
.await?
|
||||
.1
|
||||
.maybe_fallback(None, job.concurrent_limit, job.concurrency_time_window_s);
|
||||
if has_active_concurrency_limit(settings.concurrent_limit)
|
||||
&& !*DISABLE_CONCURRENCY_LIMIT
|
||||
&& job.canceled_by.is_none()
|
||||
{
|
||||
let key = concurrency_key(db, &job.id).await?.ok_or_else(|| {
|
||||
Error::internal_err(format!("No concurrency key found for job {}", job.id))
|
||||
})?;
|
||||
if !key.is_empty() {
|
||||
return Ok(crate::jobs_ee::update_concurrency_counter(
|
||||
db,
|
||||
&job.id,
|
||||
key,
|
||||
serde_json::json!({ job.id.to_string(): {} }),
|
||||
job.id.to_string(),
|
||||
settings.concurrency_time_window_s.unwrap_or(0),
|
||||
settings.concurrent_limit.unwrap_or_default(),
|
||||
)
|
||||
.await?
|
||||
.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(not(all(feature = "private", feature = "enterprise")))]
|
||||
let _ = (db, job);
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
pub async fn custom_concurrency_key(
|
||||
db: &Pool<Postgres>,
|
||||
job_id: &Uuid,
|
||||
|
||||
@@ -7,16 +7,15 @@ use crate::ai::utils::{
|
||||
use crate::common::OccupancyMetrics;
|
||||
use crate::result_processor::handle_non_flow_job_error;
|
||||
use crate::worker_flow::{
|
||||
evaluate_input_transform, raw_script_to_payload, script_to_payload, JobPayloadWithTag,
|
||||
};
|
||||
use crate::{
|
||||
create_job_dir, handle_queued_job, JobCompletedReceiver, JobCompletedSender, SendResult,
|
||||
SendResultPayload,
|
||||
evaluate_input_transform, raw_script_to_payload, resolve_flow_step_tag, script_to_payload,
|
||||
JobPayloadWithTag,
|
||||
};
|
||||
use crate::{create_job_dir, handle_queued_job, JobCompletedSender};
|
||||
use anyhow::Context;
|
||||
use mappable_rc::Marc;
|
||||
use serde_json::value::RawValue;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use sqlx::types::Json;
|
||||
use std::{collections::HashMap, sync::Arc, time::Duration};
|
||||
use uuid::Uuid;
|
||||
use windmill_ai::{ai_types::OpenAIToolCall, query_builder::StreamEventSink, types::*};
|
||||
use windmill_common::jobs::JobPayload;
|
||||
@@ -36,11 +35,12 @@ use windmill_common::{
|
||||
flow_conversations::{MessageExtras, MessageType},
|
||||
flow_status::AgentAction,
|
||||
flows::FlowModuleValue,
|
||||
worker::{to_raw_value, Connection},
|
||||
worker::{make_tool_job_pull_query, to_raw_value, Connection, WORKER_CONFIG},
|
||||
};
|
||||
use windmill_queue::{
|
||||
add_completed_job, add_completed_job_error, check_tag_available_for_push, get_mini_pulled_job,
|
||||
push, resolve_push_tag, MiniCompletedJob, MiniPulledJob, PushArgs, PushIsolationLevel,
|
||||
append_logs, check_tag_available_for_push, get_mini_pulled_job, pull, push, resolve_push_tag,
|
||||
tag_reads_flow_expr, try_admit_owned_job, MiniCompletedJob, MiniPulledJob, PushArgs,
|
||||
PushIsolationLevel,
|
||||
};
|
||||
|
||||
/// Shared collection of abort handles for spawned tool tasks.
|
||||
@@ -74,6 +74,8 @@ pub struct ToolExecutionContext<'a> {
|
||||
pub stream_event_processor: Option<&'a StreamEventProcessor>,
|
||||
pub flow_context: &'a mut FlowContext,
|
||||
pub omit_output_from_conversation: bool,
|
||||
/// The agent's flow `preserve_step_tags`: its tools pick their tag as that flow's steps do.
|
||||
pub preserve_step_tags: bool,
|
||||
/// The thinking that led to this round's calls, stored on the first tool row written.
|
||||
/// None when the round wrote text, whose row carries it.
|
||||
pub reasoning: Option<String>,
|
||||
@@ -82,6 +84,7 @@ pub struct ToolExecutionContext<'a> {
|
||||
|
||||
// Abort handles for spawned tool tasks (used for force-cancel cleanup)
|
||||
pub tool_abort_handles: ToolAbortHandles,
|
||||
pub job_completed_tx: JobCompletedSender,
|
||||
}
|
||||
|
||||
/// Execute all tool calls from an AI response
|
||||
@@ -98,7 +101,8 @@ pub async fn execute_tool_calls(
|
||||
let mut used_structured_output_tool = false;
|
||||
let mut final_content = None;
|
||||
|
||||
for tool_call in tool_calls.iter() {
|
||||
let mut calls = tool_calls.iter().peekable();
|
||||
while let Some(tool_call) = calls.next() {
|
||||
// Stream tool call progress
|
||||
if let Some(stream_event_processor) = ctx.stream_event_processor {
|
||||
let event = StreamingEvent::ToolExecution {
|
||||
@@ -150,15 +154,24 @@ pub async fn execute_tool_calls(
|
||||
)
|
||||
.await?;
|
||||
} else if tool.module.is_some() {
|
||||
execute_windmill_tool(
|
||||
&mut ctx,
|
||||
tool_call,
|
||||
tool,
|
||||
actions,
|
||||
&mut messages,
|
||||
final_events_str,
|
||||
)
|
||||
.await?;
|
||||
let mut batch = vec![(tool_call, tool)];
|
||||
while let Some(next_call) = calls.peek() {
|
||||
if structured_output_tool_name.as_deref()
|
||||
== Some(next_call.function.name.as_str())
|
||||
{
|
||||
break;
|
||||
}
|
||||
let Some(next_tool) = tools.iter().find(|t| {
|
||||
t.def.function.name == next_call.function.name
|
||||
&& t.mcp_source.is_none()
|
||||
&& t.module.is_some()
|
||||
}) else {
|
||||
break;
|
||||
};
|
||||
batch.push((calls.next().unwrap(), next_tool));
|
||||
}
|
||||
execute_windmill_tools(&mut ctx, &batch, actions, &mut messages, final_events_str)
|
||||
.await?;
|
||||
} else {
|
||||
return Err(Error::internal_err(format!(
|
||||
"Tool type not supported: {}",
|
||||
@@ -313,14 +326,13 @@ async fn execute_mcp_tool_call(
|
||||
}
|
||||
|
||||
/// Execute a Windmill tool (script or flow)
|
||||
async fn execute_windmill_tool(
|
||||
ctx: &mut ToolExecutionContext<'_>,
|
||||
async fn enqueue_windmill_tool(
|
||||
ctx: &ToolExecutionContext<'_>,
|
||||
tool_call: &OpenAIToolCall,
|
||||
tool: &Tool,
|
||||
actions: &mut Vec<AgentAction>,
|
||||
messages: &mut Vec<OpenAIMessage>,
|
||||
final_events_str: &mut String,
|
||||
) -> Result<(), Error> {
|
||||
reserved: bool,
|
||||
) -> Result<(Uuid, bool), Error> {
|
||||
// Regular Windmill tools must have a module
|
||||
let tool_module = tool.module.as_ref().ok_or_else(|| {
|
||||
Error::internal_err(format!("Tool {} has no module", tool_call.function.name))
|
||||
@@ -402,8 +414,6 @@ async fn execute_windmill_tool(
|
||||
tool_call_args.insert(key.clone(), result);
|
||||
}
|
||||
|
||||
let is_ai_agent_tool = matches!(tool_value, FlowModuleValue::AIAgent { .. });
|
||||
|
||||
let job_payload = match tool_value {
|
||||
FlowModuleValue::Script { path: script_path, hash: script_hash, tag_override, .. } => {
|
||||
script_to_payload(
|
||||
@@ -473,7 +483,6 @@ async fn execute_windmill_tool(
|
||||
));
|
||||
}
|
||||
let path = format!("{}/tools/{}", ctx.job.runnable_path(), tool_module.id);
|
||||
// a nested agent always runs inline on this worker, under this agent's tag
|
||||
JobPayloadWithTag {
|
||||
payload: JobPayload::AIAgent { path },
|
||||
tag: None,
|
||||
@@ -502,18 +511,34 @@ async fn execute_windmill_tool(
|
||||
|
||||
let push_args = PushArgs { args: &tool_call_args, extra: None };
|
||||
|
||||
// A tool with a tag of its own naming another queue than this agent's must run on a worker
|
||||
// that tag selects, and like any tag the caller chose it must pass the workspace's custom tag
|
||||
// restrictions. It runs inline when this worker serves that queue: the agent holds this
|
||||
// worker while it waits, so dispatching to a queue it serves could wait on itself.
|
||||
let mut resolved_tool_tag = None;
|
||||
if let Some(tag) = job_payload.tag.as_deref() {
|
||||
resolved_tool_tag = resolve_push_tag(tag, &push_args, &ctx.job.workspace_id, ctx.db)
|
||||
.await
|
||||
.filter(|resolved| *resolved != ctx.job.tag);
|
||||
// The agent's tag stands in for the flow's, as a sub-flow's does for its own steps.
|
||||
let tool_tag = resolve_flow_step_tag(
|
||||
false,
|
||||
&ctx.job.tag,
|
||||
&ctx.job.workspace_id,
|
||||
ctx.preserve_step_tags,
|
||||
job_payload.tag.as_deref(),
|
||||
);
|
||||
if tool_tag.as_deref().is_some_and(tag_reads_flow_expr) {
|
||||
append_logs(
|
||||
&ctx.job.id,
|
||||
&ctx.job.workspace_id,
|
||||
format!(
|
||||
"Tool '{}' is tagged with `$flow_expr[...]`, which AI agent tools do not resolve, so it runs on its default tag.\n",
|
||||
tool_call.function.name
|
||||
),
|
||||
ctx.conn,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
let tool_tag = match (job_payload.tag.as_deref(), resolved_tool_tag.as_ref()) {
|
||||
(Some(tag), Some(_)) => {
|
||||
|
||||
// A tool's own tag routes its job, so like any tag the caller chose it must pass the
|
||||
// workspace's custom tag restrictions. The agent's tag was already checked when it was pushed.
|
||||
if let Some(tag) = tool_tag.as_deref() {
|
||||
if resolve_push_tag(tag, &push_args, &ctx.job.workspace_id, ctx.db)
|
||||
.await
|
||||
.is_some_and(|resolved| resolved != ctx.job.tag)
|
||||
{
|
||||
let is_super_admin = windmill_common::auth::is_super_admin_email(ctx.db, email).await?;
|
||||
check_tag_available_for_push(
|
||||
ctx.db,
|
||||
@@ -531,16 +556,8 @@ async fn execute_windmill_tool(
|
||||
)),
|
||||
e => e,
|
||||
})?;
|
||||
Some(tag.to_string())
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let run_inline = resolved_tool_tag.is_none_or(|resolved| {
|
||||
windmill_common::worker::WORKER_CONFIG
|
||||
.load()
|
||||
.worker_tags
|
||||
.contains(&resolved)
|
||||
});
|
||||
}
|
||||
|
||||
let mut tx = ctx.db.begin().await?;
|
||||
|
||||
@@ -573,77 +590,71 @@ async fn execute_windmill_tool(
|
||||
false,
|
||||
None,
|
||||
ctx.job.visible_to_owner,
|
||||
Some(tool_tag.unwrap_or_else(|| ctx.job.tag.clone())),
|
||||
tool_tag,
|
||||
job_payload.timeout,
|
||||
None,
|
||||
job_priority,
|
||||
job_perms.as_ref(),
|
||||
run_inline,
|
||||
false,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
tx.commit().await?;
|
||||
|
||||
if !run_inline {
|
||||
// Like an inline tool's failure, a lost wait reaches the model rather than failing the
|
||||
// agent.
|
||||
let (result, success) =
|
||||
match wait_for_dispatched_tool_job(ctx.db, &uuid, &ctx.job.workspace_id).await {
|
||||
Ok(outcome) => outcome,
|
||||
Err(e) => (
|
||||
to_raw_value(&serde_json::json!({ "error": { "message": e.to_string() } })),
|
||||
false,
|
||||
),
|
||||
};
|
||||
return report_tool_result(
|
||||
ctx,
|
||||
tool_call,
|
||||
tool_module,
|
||||
job_id,
|
||||
&result,
|
||||
success,
|
||||
is_ai_agent_tool && success,
|
||||
messages,
|
||||
final_events_str,
|
||||
let mut tx = tx;
|
||||
let reserved = if reserved {
|
||||
let tags = local_tool_tags();
|
||||
// Reserve a supported first tool before commit so other workers cannot race it.
|
||||
let claimed = sqlx::query!(
|
||||
"UPDATE v2_job_queue SET worker = $1, running = true, started_at = now()
|
||||
WHERE id = $2 AND tag = ANY($3)",
|
||||
ctx.worker_name,
|
||||
uuid,
|
||||
&tags,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
let tool_job = get_mini_pulled_job(ctx.db, &uuid).await?;
|
||||
|
||||
let Some(tool_job) = tool_job else {
|
||||
return Err(Error::internal_err("Tool job not found".to_string()));
|
||||
.execute(&mut *tx)
|
||||
.await?
|
||||
.rows_affected()
|
||||
== 1;
|
||||
if claimed {
|
||||
sqlx::query!("UPDATE v2_job_runtime SET ping = now() WHERE id = $1", uuid)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
claimed
|
||||
} else {
|
||||
false
|
||||
};
|
||||
tx.commit().await?;
|
||||
Ok((uuid, reserved))
|
||||
}
|
||||
|
||||
// The tool gets a token of its own, as it would on any worker: the agent's token would
|
||||
// identify the tool as the agent job (its OIDC path and job id).
|
||||
let tool_client = AuthedClient::new(
|
||||
ctx.base_internal_url.to_string(),
|
||||
tool_job.workspace_id.clone(),
|
||||
windmill_queue::create_token(ctx.db, &tool_job, None).await,
|
||||
None,
|
||||
);
|
||||
let tool_job = Arc::new(tool_job);
|
||||
fn local_tool_tags() -> Vec<String> {
|
||||
WORKER_CONFIG
|
||||
.load()
|
||||
.priority_tags_sorted
|
||||
.iter()
|
||||
.flat_map(|group| group.tags.iter().cloned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
let (inner_job_completed_tx, inner_job_completed_rx) = JobCompletedSender::new(ctx.conn, 1);
|
||||
|
||||
let inner_job_completed_rx = inner_job_completed_rx.expect(
|
||||
"inner_job_completed_tx should be set as agent jobs are not supported on agent workers",
|
||||
);
|
||||
fn spawn_local_tool(
|
||||
ctx: &ToolExecutionContext<'_>,
|
||||
tool_job: MiniPulledJob,
|
||||
reserved: bool,
|
||||
) -> tokio::task::JoinHandle<Result<OccupancyMetrics, Error>> {
|
||||
let mut tool_job = Arc::new(tool_job);
|
||||
|
||||
// Spawn handle_queued_job on separate task to prevent tokio stack overflow
|
||||
// Clone everything needed for the spawned task
|
||||
let tool_job_spawn = tool_job.clone();
|
||||
let db = ctx.db.clone();
|
||||
let conn_spawn = ctx.conn.clone();
|
||||
let client_spawn = tool_client;
|
||||
let hostname_spawn = ctx.hostname.to_string();
|
||||
let worker_name_spawn = ctx.worker_name.to_string();
|
||||
let worker_dir_spawn = ctx.worker_dir.to_string();
|
||||
let base_internal_url_spawn = ctx.base_internal_url.to_string();
|
||||
let inner_job_completed_tx_spawn = inner_job_completed_tx.clone();
|
||||
let job_completed_tx = ctx.job_completed_tx.clone();
|
||||
let mut occupancy_metrics_spawn = ctx.occupancy_metrics.clone();
|
||||
let mut killpill_rx_spawn = ctx.killpill_rx.resubscribe();
|
||||
|
||||
@@ -652,147 +663,270 @@ async fn execute_windmill_tool(
|
||||
#[cfg(feature = "benchmark")]
|
||||
let mut bench_spawn = windmill_common::bench::BenchmarkIter::new();
|
||||
|
||||
let job_dir = create_job_dir(&worker_dir_spawn, tool_job_spawn.id).await;
|
||||
let result = async {
|
||||
if reserved {
|
||||
loop {
|
||||
let queued = get_mini_pulled_job(&db, &tool_job.id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::AlreadyCompleted("Tool job already completed".to_string()))?;
|
||||
tool_job = Arc::new(queued);
|
||||
sqlx::query!(
|
||||
"UPDATE v2_job_runtime SET ping = now() WHERE id = $1",
|
||||
tool_job.id,
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
if try_admit_owned_job(&db, &tool_job).await? {
|
||||
let started_at = sqlx::query_scalar!(
|
||||
"UPDATE v2_job_queue SET started_at = now() WHERE id = $1 RETURNING started_at",
|
||||
tool_job.id,
|
||||
)
|
||||
.fetch_optional(&db)
|
||||
.await?
|
||||
.flatten();
|
||||
Arc::make_mut(&mut tool_job).started_at = started_at;
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
let perms =
|
||||
windmill_common::auth::get_job_perms(&db, &tool_job.id, &tool_job.workspace_id).await?;
|
||||
let token = windmill_queue::create_token(&db, &tool_job, perms).await;
|
||||
let client_spawn = AuthedClient::new(
|
||||
base_internal_url_spawn.clone(),
|
||||
tool_job.workspace_id.clone(),
|
||||
token,
|
||||
None,
|
||||
);
|
||||
let job_dir = create_job_dir(&worker_dir_spawn, tool_job.id).await;
|
||||
|
||||
let result = handle_queued_job(
|
||||
tool_job_spawn,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
&conn_spawn,
|
||||
&client_spawn,
|
||||
&hostname_spawn,
|
||||
&worker_name_spawn,
|
||||
&worker_dir_spawn,
|
||||
&job_dir,
|
||||
None,
|
||||
&base_internal_url_spawn,
|
||||
inner_job_completed_tx_spawn,
|
||||
&mut occupancy_metrics_spawn,
|
||||
&mut killpill_rx_spawn,
|
||||
None,
|
||||
None,
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench_spawn,
|
||||
)
|
||||
handle_queued_job(
|
||||
tool_job.clone(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
&conn_spawn,
|
||||
&client_spawn,
|
||||
&hostname_spawn,
|
||||
&worker_name_spawn,
|
||||
&worker_dir_spawn,
|
||||
&job_dir,
|
||||
None,
|
||||
&base_internal_url_spawn,
|
||||
job_completed_tx,
|
||||
&mut occupancy_metrics_spawn,
|
||||
&mut killpill_rx_spawn,
|
||||
None,
|
||||
None,
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench_spawn,
|
||||
)
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
|
||||
// Return both result and updated metrics
|
||||
(result, occupancy_metrics_spawn)
|
||||
match result {
|
||||
Err(err) => {
|
||||
let err_string = format!("{}: {}", err.name(), err);
|
||||
handle_non_flow_job_error(
|
||||
&db,
|
||||
&MiniCompletedJob::from(tool_job),
|
||||
0,
|
||||
None,
|
||||
err_string,
|
||||
windmill_common::worker::error_to_value(&err),
|
||||
&worker_name_spawn,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(_) => {}
|
||||
}
|
||||
Ok(occupancy_metrics_spawn)
|
||||
});
|
||||
|
||||
// Register abort handle so the task can be killed on force-cancel
|
||||
let abort_handle = join_handle.abort_handle();
|
||||
// unwrap safe: lock is only held briefly for push/drain, no panic possible inside
|
||||
ctx.tool_abort_handles.lock().unwrap().push(abort_handle);
|
||||
|
||||
// Await the spawned task
|
||||
let (handle_result, updated_occupancy) = join_handle.await.map_err(|e| {
|
||||
if e.is_cancelled() {
|
||||
Error::ExecutionErr("Tool execution task was cancelled".to_string())
|
||||
} else {
|
||||
Error::internal_err(format!("Tool execution task failed: {}", e))
|
||||
}
|
||||
})?;
|
||||
|
||||
// Merge occupancy metrics back
|
||||
ctx.occupancy_metrics.total_duration_of_running_jobs =
|
||||
updated_occupancy.total_duration_of_running_jobs;
|
||||
|
||||
match handle_result {
|
||||
Err(err) => {
|
||||
handle_tool_execution_error(
|
||||
ctx,
|
||||
tool_call,
|
||||
tool_module,
|
||||
&MiniCompletedJob::from(tool_job),
|
||||
job_id,
|
||||
err,
|
||||
messages,
|
||||
final_events_str,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
Ok(outcome) => {
|
||||
handle_tool_execution_success(
|
||||
ctx,
|
||||
tool_call,
|
||||
tool_module,
|
||||
job_id,
|
||||
outcome.is_success(),
|
||||
is_ai_agent_tool,
|
||||
inner_job_completed_rx,
|
||||
messages,
|
||||
final_events_str,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
join_handle
|
||||
}
|
||||
|
||||
/// Handle tool execution error
|
||||
async fn handle_tool_execution_error(
|
||||
async fn execute_windmill_tools(
|
||||
ctx: &mut ToolExecutionContext<'_>,
|
||||
tool_call: &OpenAIToolCall,
|
||||
tool_module: &windmill_common::flows::FlowModule,
|
||||
tool_job: &MiniCompletedJob,
|
||||
job_id: Uuid,
|
||||
err: Error,
|
||||
batch: &[(&OpenAIToolCall, &Tool)],
|
||||
actions: &mut Vec<AgentAction>,
|
||||
messages: &mut Vec<OpenAIMessage>,
|
||||
final_events_str: &mut String,
|
||||
) -> Result<(), Error> {
|
||||
let err_string = format!("{}: {}", err.name(), err.to_string());
|
||||
let err_json = windmill_common::worker::error_to_value(&err);
|
||||
let _ = handle_non_flow_job_error(
|
||||
ctx.db,
|
||||
tool_job,
|
||||
0,
|
||||
None,
|
||||
err_string.clone(),
|
||||
err_json,
|
||||
ctx.worker_name,
|
||||
)
|
||||
.await;
|
||||
|
||||
let error_message = format!("Error running tool: {}", err_string);
|
||||
messages.push(OpenAIMessage {
|
||||
role: "tool".to_string(),
|
||||
content: Some(OpenAIContent::Text(error_message.clone())),
|
||||
tool_call_id: Some(tool_call.id.clone()),
|
||||
agent_action: Some(AgentAction::ToolCall {
|
||||
job_id,
|
||||
function_name: tool_call.function.name.clone(),
|
||||
module_id: tool_module.id.clone(),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
// Stream tool result (error case)
|
||||
if let Some(stream_event_processor) = ctx.stream_event_processor {
|
||||
let tool_result_event = StreamingEvent::ToolResult {
|
||||
call_id: tool_call.id.clone(),
|
||||
function_name: tool_call.function.name.clone(),
|
||||
result: error_message.clone(),
|
||||
success: false,
|
||||
};
|
||||
stream_event_processor
|
||||
.send(tool_result_event, final_events_str)
|
||||
.await?;
|
||||
let mut job_ids = Vec::with_capacity(batch.len());
|
||||
let mut local = None;
|
||||
for (index, (call, tool)) in batch.iter().enumerate() {
|
||||
if index > 0 {
|
||||
if let Some(processor) = ctx.stream_event_processor {
|
||||
processor
|
||||
.send(
|
||||
StreamingEvent::ToolExecution {
|
||||
call_id: call.id.clone(),
|
||||
function_name: call.function.name.clone(),
|
||||
},
|
||||
final_events_str,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
let (id, reserved) = enqueue_windmill_tool(ctx, call, tool, actions, index == 0).await?;
|
||||
job_ids.push(id);
|
||||
if reserved {
|
||||
let job = get_mini_pulled_job(ctx.db, &id)
|
||||
.await?
|
||||
.ok_or_else(|| Error::internal_err("Reserved tool job not found".to_string()))?;
|
||||
local = Some(spawn_local_tool(ctx, job, true));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(parent_job) = ctx.parent_job {
|
||||
update_flow_status_module_with_actions_success(ctx.db, parent_job, false).await?;
|
||||
let mut results: HashMap<Uuid, (bool, String)> = HashMap::new();
|
||||
let mut next_result = 0;
|
||||
let mut poll = tokio::time::interval(Duration::from_millis(100));
|
||||
poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
|
||||
while next_result < batch.len() || local.is_some() {
|
||||
tokio::select! {
|
||||
result = async { local.as_mut().unwrap().await }, if local.is_some() => {
|
||||
local = None;
|
||||
let metrics = result.map_err(|e| Error::internal_err(format!("Tool task failed: {e}")))??;
|
||||
ctx.occupancy_metrics.total_duration_of_running_jobs = metrics.total_duration_of_running_jobs;
|
||||
ctx.tool_abort_handles.lock().unwrap().retain(|handle| !handle.is_finished());
|
||||
}
|
||||
_ = poll.tick() => {}
|
||||
}
|
||||
|
||||
let pending: Vec<Uuid> = job_ids[next_result..]
|
||||
.iter()
|
||||
.filter(|id| !results.contains_key(id))
|
||||
.copied()
|
||||
.collect();
|
||||
if !pending.is_empty() {
|
||||
let completed = sqlx::query!(
|
||||
"SELECT id, status = 'success' AS \"success!\", result AS \"result: Json<Box<RawValue>>\"
|
||||
FROM v2_job_completed WHERE workspace_id = $1 AND id = ANY($2)",
|
||||
ctx.job.workspace_id,
|
||||
&pending,
|
||||
).fetch_all(ctx.db).await?;
|
||||
for completed in completed {
|
||||
let index = job_ids
|
||||
.iter()
|
||||
.position(|id| *id == completed.id)
|
||||
.ok_or_else(|| Error::internal_err("Unexpected tool completion".to_string()))?;
|
||||
let (call, tool) = batch[index];
|
||||
let result = completed
|
||||
.result
|
||||
.map(|value| value.0)
|
||||
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
|
||||
let is_agent = tool.module.as_ref().is_some_and(|module| {
|
||||
matches!(module.get_value(), Ok(FlowModuleValue::AIAgent { .. }))
|
||||
});
|
||||
let content = if is_agent && completed.success {
|
||||
extract_ai_agent_output(&result).unwrap_or_else(|| result.get().to_string())
|
||||
} else {
|
||||
result.get().to_string()
|
||||
};
|
||||
if let Some(processor) = ctx.stream_event_processor {
|
||||
processor
|
||||
.send(
|
||||
StreamingEvent::ToolResult {
|
||||
call_id: call.id.clone(),
|
||||
function_name: call.function.name.clone(),
|
||||
result: content.clone(),
|
||||
success: completed.success,
|
||||
},
|
||||
final_events_str,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
results.insert(completed.id, (completed.success, content));
|
||||
}
|
||||
}
|
||||
|
||||
// Transcript rows, model messages, and positional action statuses share call order.
|
||||
while next_result < batch.len() {
|
||||
let job_id = job_ids[next_result];
|
||||
let Some((success, content)) = results.remove(&job_id) else {
|
||||
break;
|
||||
};
|
||||
let (call, tool) = batch[next_result];
|
||||
let module = tool
|
||||
.module
|
||||
.as_ref()
|
||||
.ok_or_else(|| Error::internal_err("Windmill tool has no module".to_string()))?;
|
||||
messages.push(OpenAIMessage {
|
||||
role: "tool".to_string(),
|
||||
content: Some(OpenAIContent::Text(content.clone())),
|
||||
tool_call_id: Some(call.id.clone()),
|
||||
agent_action: Some(AgentAction::ToolCall {
|
||||
job_id,
|
||||
function_name: call.function.name.clone(),
|
||||
module_id: module.id.clone(),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
if let Some(parent) = ctx.parent_job {
|
||||
update_flow_status_module_with_actions_success(ctx.db, parent, success).await?;
|
||||
}
|
||||
let (content, extras) = windmill_tool_row(call, success, &content);
|
||||
add_tool_message_to_chat(ctx, Some(job_id), &content, success, Some(extras)).await;
|
||||
next_result += 1;
|
||||
}
|
||||
|
||||
if local.is_none() && next_result < batch.len() {
|
||||
let pending: Vec<Uuid> = job_ids[next_result..]
|
||||
.iter()
|
||||
.filter(|id| !results.contains_key(id))
|
||||
.copied()
|
||||
.collect();
|
||||
if !pending.is_empty() {
|
||||
local = claim_local_tool(ctx, &pending).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let (content, extras) = windmill_tool_row(tool_call, false, &error_message);
|
||||
add_tool_message_to_chat(ctx, Some(job_id), &content, false, Some(extras)).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn claim_local_tool(
|
||||
ctx: &ToolExecutionContext<'_>,
|
||||
pending: &[Uuid],
|
||||
) -> Result<Option<tokio::task::JoinHandle<Result<OccupancyMetrics, Error>>>, Error> {
|
||||
let query = (
|
||||
String::new(),
|
||||
make_tool_job_pull_query(pending, &local_tool_tags()),
|
||||
);
|
||||
#[cfg(feature = "benchmark")]
|
||||
let mut bench = windmill_common::bench::BenchmarkIter::new();
|
||||
let mut pulled = pull(
|
||||
ctx.db,
|
||||
false,
|
||||
ctx.worker_name,
|
||||
Some(&query),
|
||||
#[cfg(feature = "benchmark")]
|
||||
&mut bench,
|
||||
)
|
||||
.await?;
|
||||
if let Err(err) = pulled.maybe_apply_debouncing(ctx.db).await {
|
||||
pulled.error_while_preprocessing = Some(err.to_string());
|
||||
}
|
||||
match pulled.to_pulled_job() {
|
||||
Ok(job) => Ok(job.map(|job| spawn_local_tool(ctx, job.job, false))),
|
||||
Err(
|
||||
windmill_queue::PulledJobResultToJobErr::MissingConcurrencyKey(job)
|
||||
| windmill_queue::PulledJobResultToJobErr::ErrorWhilePreprocessing(job),
|
||||
) => {
|
||||
ctx.job_completed_tx.send_job(job, true).await?;
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the `output` field of an `AIAgentResult` envelope, serialized back to JSON.
|
||||
/// Returns `None` if the result is not a JSON object carrying an `output` field.
|
||||
fn extract_ai_agent_output(result: &RawValue) -> Option<String> {
|
||||
@@ -802,200 +936,6 @@ fn extract_ai_agent_output(result: &RawValue) -> Option<String> {
|
||||
.map(|output| output.get().to_string())
|
||||
}
|
||||
|
||||
/// Handle tool execution success
|
||||
async fn handle_tool_execution_success(
|
||||
ctx: &mut ToolExecutionContext<'_>,
|
||||
tool_call: &OpenAIToolCall,
|
||||
tool_module: &windmill_common::flows::FlowModule,
|
||||
job_id: Uuid,
|
||||
success: bool,
|
||||
is_ai_agent_tool: bool,
|
||||
inner_job_completed_rx: JobCompletedReceiver,
|
||||
messages: &mut Vec<OpenAIMessage>,
|
||||
final_events_str: &mut String,
|
||||
) -> Result<(), Error> {
|
||||
let send_result = inner_job_completed_rx.bounded_rx.try_recv().ok();
|
||||
|
||||
let (result, job_success) = if let Some(SendResult {
|
||||
result: SendResultPayload::JobCompleted(ref jc),
|
||||
..
|
||||
}) = send_result
|
||||
{
|
||||
let result = jc.result.clone();
|
||||
// Write tool completion to the DB inline instead of forwarding through
|
||||
// the parent channel. Forwarding would deadlock for nested agents: the
|
||||
// sub-tool result would fill the parent's bounded(1) channel, leaving
|
||||
// no room for the agent's own completion from process_result.
|
||||
if jc.success {
|
||||
add_completed_job(
|
||||
ctx.db,
|
||||
&jc.job,
|
||||
true,
|
||||
false,
|
||||
sqlx::types::Json(&*jc.result),
|
||||
jc.result_columns.clone(),
|
||||
jc.mem_peak,
|
||||
jc.canceled_by.clone(),
|
||||
false,
|
||||
jc.duration,
|
||||
jc.from_cache.unwrap_or(false),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::internal_err(format!("Failed to add completed job: {e}")))?;
|
||||
} else {
|
||||
let error_value: serde_json::Value =
|
||||
serde_json::from_str(jc.result.get()).unwrap_or_else(|_| {
|
||||
serde_json::json!({ "message": format!("Non serializable error: {}", jc.result.get()) })
|
||||
});
|
||||
add_completed_job_error(
|
||||
ctx.db,
|
||||
&jc.job,
|
||||
jc.mem_peak,
|
||||
jc.canceled_by.clone(),
|
||||
error_value,
|
||||
ctx.worker_name,
|
||||
false,
|
||||
jc.duration,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| Error::internal_err(format!("Failed to add completed job error: {e}")))?;
|
||||
}
|
||||
(result, jc.success)
|
||||
} else {
|
||||
return Err(Error::internal_err(
|
||||
"Tool job completed but no result".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
report_tool_result(
|
||||
ctx,
|
||||
tool_call,
|
||||
tool_module,
|
||||
job_id,
|
||||
&result,
|
||||
success,
|
||||
is_ai_agent_tool && job_success,
|
||||
messages,
|
||||
final_events_str,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Wait for a tool job another worker runs; returns its result and whether it succeeded.
|
||||
/// Canceling the agent cancels this job along with it, as a child of the agent's job.
|
||||
async fn wait_for_dispatched_tool_job(
|
||||
db: &DB,
|
||||
id: &Uuid,
|
||||
w_id: &str,
|
||||
) -> Result<(Box<RawValue>, bool), Error> {
|
||||
const MAX_CONSECUTIVE_POLL_ERRORS: u32 = 10;
|
||||
let mut interval = std::time::Duration::from_millis(50);
|
||||
let mut poll_errors = 0;
|
||||
loop {
|
||||
// One statement reads both tables from one snapshot, so a job completing between two
|
||||
// reads cannot look like one that vanished.
|
||||
let state = match sqlx::query!(
|
||||
"SELECT c.id IS NOT NULL AS \"completed!\",
|
||||
c.result AS \"result: sqlx::types::Json<Box<RawValue>>\",
|
||||
c.status = 'success' AS \"success\",
|
||||
EXISTS(SELECT 1 FROM v2_job_queue WHERE id = $1) AS \"queued!\"
|
||||
FROM (SELECT 1) one
|
||||
LEFT JOIN v2_job_completed c ON c.id = $1 AND c.workspace_id = $2",
|
||||
id,
|
||||
w_id
|
||||
)
|
||||
.fetch_one(db)
|
||||
.await
|
||||
{
|
||||
Ok(state) => {
|
||||
poll_errors = 0;
|
||||
state
|
||||
}
|
||||
Err(e) if poll_errors < MAX_CONSECUTIVE_POLL_ERRORS => {
|
||||
poll_errors += 1;
|
||||
tracing::warn!("polling tool job {id} failed, retrying: {e}");
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
continue;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
};
|
||||
if state.completed {
|
||||
let result = state
|
||||
.result
|
||||
.map(|r| r.0)
|
||||
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
|
||||
return Ok((result, state.success.unwrap_or(false)));
|
||||
}
|
||||
if !state.queued {
|
||||
return Err(Error::internal_err(format!(
|
||||
"tool job {id} is neither queued nor completed"
|
||||
)));
|
||||
}
|
||||
tokio::time::sleep(interval).await;
|
||||
interval = std::cmp::min(interval * 2, std::time::Duration::from_secs(1));
|
||||
}
|
||||
}
|
||||
|
||||
/// Hand a finished tool job's result to the model, the stream, the flow status and the chat.
|
||||
async fn report_tool_result(
|
||||
ctx: &mut ToolExecutionContext<'_>,
|
||||
tool_call: &OpenAIToolCall,
|
||||
tool_module: &windmill_common::flows::FlowModule,
|
||||
job_id: Uuid,
|
||||
result: &RawValue,
|
||||
success: bool,
|
||||
is_ai_agent_output: bool,
|
||||
messages: &mut Vec<OpenAIMessage>,
|
||||
final_events_str: &mut String,
|
||||
) -> Result<(), Error> {
|
||||
// A nested agent returns the whole `AIAgentResult` envelope: on top of `output` it carries
|
||||
// the child's entire message history, stream log and token usage. Feeding that back would
|
||||
// grow the caller's context by the child's full transcript on every call, so the caller only
|
||||
// sees `output`. The envelope stays intact in the tool job's completed row.
|
||||
let tool_result = if is_ai_agent_output {
|
||||
extract_ai_agent_output(result).unwrap_or_else(|| result.get().to_string())
|
||||
} else {
|
||||
result.get().to_string()
|
||||
};
|
||||
|
||||
messages.push(OpenAIMessage {
|
||||
role: "tool".to_string(),
|
||||
content: Some(OpenAIContent::Text(tool_result.clone())),
|
||||
tool_call_id: Some(tool_call.id.clone()),
|
||||
agent_action: Some(AgentAction::ToolCall {
|
||||
job_id,
|
||||
function_name: tool_call.function.name.clone(),
|
||||
module_id: tool_module.id.clone(),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let (content, extras) = windmill_tool_row(tool_call, success, &tool_result);
|
||||
|
||||
// The job ran; whether it ran successfully is `success`, and the row stored below is
|
||||
// worded from it. The stream has to carry the same value, or the card the reader watches
|
||||
// and the row that replaces it describe the same call differently.
|
||||
if let Some(stream_event_processor) = ctx.stream_event_processor {
|
||||
let tool_result_event = StreamingEvent::ToolResult {
|
||||
call_id: tool_call.id.clone(),
|
||||
function_name: tool_call.function.name.clone(),
|
||||
result: tool_result,
|
||||
success,
|
||||
};
|
||||
stream_event_processor
|
||||
.send(tool_result_event, final_events_str)
|
||||
.await?;
|
||||
}
|
||||
|
||||
if let Some(parent_job) = ctx.parent_job {
|
||||
update_flow_status_module_with_actions_success(ctx.db, parent_job, success).await?;
|
||||
}
|
||||
|
||||
add_tool_message_to_chat(ctx, Some(job_id), &content, success, Some(extras)).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A Windmill tool's conversation row: worded from the tool, carrying the model's call and
|
||||
/// the exact text the model got back, the same text agent memory keeps for that tool
|
||||
/// message, so a card needs no job fetch. The call is the model's arguments, not the job's
|
||||
@@ -1060,9 +1000,7 @@ async fn add_tool_message_to_chat(
|
||||
.or(ctx.job.flow_step_id.as_deref());
|
||||
let step_name = get_step_name_from_flow(ctx.summary.as_deref(), effective_step_id);
|
||||
|
||||
// Awaited, not spawned: `created_seq` is the transcript's order, so a round's rows
|
||||
// must commit in the order of its calls. Calls run one after another; running them
|
||||
// in parallel would need their rows written in call order all the same.
|
||||
// created_seq defines transcript order, so these writes must stay sequential.
|
||||
if let Err(e) = add_message_to_conversation(
|
||||
ctx.db,
|
||||
&memory_id,
|
||||
|
||||
@@ -607,6 +607,7 @@ pub async fn handle_ai_agent_job(
|
||||
hostname: &str,
|
||||
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
|
||||
has_stream: &mut bool,
|
||||
job_completed_tx: crate::JobCompletedSender,
|
||||
) -> Result<Box<RawValue>, Error> {
|
||||
// build_args_map returns None if no $res:/$var: transforms needed, in which case use original args
|
||||
let local_args = match build_args_map(job, client, conn).await? {
|
||||
@@ -686,6 +687,7 @@ pub async fn handle_ai_agent_job(
|
||||
};
|
||||
|
||||
let value = flow_data.value();
|
||||
let preserve_step_tags = value.preserve_step_tags;
|
||||
|
||||
let module = if direct_parent_job_kind == JobKind::AIAgent {
|
||||
let parent_agent_step_id = direct_parent_job_flow_step_id.as_deref().ok_or_else(|| {
|
||||
@@ -1111,8 +1113,10 @@ pub async fn handle_ai_agent_job(
|
||||
has_stream,
|
||||
has_websearch,
|
||||
omit_output_from_conversation,
|
||||
preserve_step_tags,
|
||||
cancel_rx,
|
||||
tool_abort_handles.clone(),
|
||||
job_completed_tx,
|
||||
);
|
||||
|
||||
let mut occupancy_opt = Some(occupancy_metrics);
|
||||
@@ -1131,13 +1135,10 @@ pub async fn handle_ai_agent_job(
|
||||
cancel_tx,
|
||||
CANCEL_GRACE_PERIOD,
|
||||
)
|
||||
.await?
|
||||
.await
|
||||
};
|
||||
// agent_fut and update_job are now dropped — borrows on mcp_clients and canceled_by released
|
||||
|
||||
// Cleanup MCP clients
|
||||
cleanup_mcp_clients(mcp_clients).await;
|
||||
|
||||
let format_cancel_info = |cb: &Option<CanceledBy>| {
|
||||
cb.as_ref()
|
||||
.map_or(("unknown".to_string(), "unknown".to_string()), |x| {
|
||||
@@ -1148,43 +1149,42 @@ pub async fn handle_ai_agent_job(
|
||||
})
|
||||
};
|
||||
|
||||
match outcome {
|
||||
GracefulPollOutcome::Ok(result) => Ok(result),
|
||||
GracefulPollOutcome::Timeout(ms) => {
|
||||
tracing::error!("AI agent timeout after {}s", ms / 1000);
|
||||
// A tool dispatched to another worker group would otherwise still run later, after
|
||||
// the agent that wanted its result is gone.
|
||||
let reason = format!("parent AI agent {} timed out", job.id);
|
||||
let cb = CanceledBy { username: None, reason: Some(reason) };
|
||||
cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, Some(cb)).await;
|
||||
Err(Error::ExecutionErr(format!(
|
||||
"AI agent timeout after (>{}s)",
|
||||
ms / 1000
|
||||
)))
|
||||
}
|
||||
GracefulPollOutcome::Cancelled { canceled_by: cb } => {
|
||||
let (by, reason) = format_cancel_info(&cb);
|
||||
Err(Error::ExecutionErr(format!(
|
||||
"Job cancelled by {by} (reason: {reason})"
|
||||
)))
|
||||
}
|
||||
GracefulPollOutcome::CancelledTimeout { canceled_by: cb } => {
|
||||
let (by, reason) = format_cancel_info(&cb);
|
||||
// Abort any still-running spawned tool tasks
|
||||
// unwrap safe: lock is only held briefly for push/drain, no panic possible inside
|
||||
for handle in tool_abort_handles.lock().unwrap().drain(..) {
|
||||
handle.abort();
|
||||
let result = match outcome {
|
||||
Err(error) => Err(error),
|
||||
Ok(outcome) => match outcome {
|
||||
GracefulPollOutcome::Ok(result) => Ok(result),
|
||||
GracefulPollOutcome::Timeout(ms) => {
|
||||
tracing::error!("AI agent timeout after {}s", ms / 1000);
|
||||
Err(Error::ExecutionErr(format!(
|
||||
"AI agent timeout after (>{}s)",
|
||||
ms / 1000
|
||||
)))
|
||||
}
|
||||
// Hard timeout: clean up orphaned jobs still stuck in v2_job_queue
|
||||
cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, cb).await;
|
||||
Err(Error::ExecutionErr(format!(
|
||||
"Job cancelled by {by} (reason: {reason}, timed out waiting for tool calls)"
|
||||
)))
|
||||
}
|
||||
GracefulPollOutcome::AlreadyCompleted => {
|
||||
Err(Error::AlreadyCompleted("Job already completed".to_string()))
|
||||
GracefulPollOutcome::Cancelled { canceled_by: cb } => {
|
||||
let (by, reason) = format_cancel_info(&cb);
|
||||
Err(Error::ExecutionErr(format!(
|
||||
"Job cancelled by {by} (reason: {reason})"
|
||||
)))
|
||||
}
|
||||
GracefulPollOutcome::CancelledTimeout { canceled_by: cb } => {
|
||||
let (by, reason) = format_cancel_info(&cb);
|
||||
Err(Error::ExecutionErr(format!(
|
||||
"Job cancelled by {by} (reason: {reason}, timed out waiting for tool calls)"
|
||||
)))
|
||||
}
|
||||
GracefulPollOutcome::AlreadyCompleted => {
|
||||
Err(Error::AlreadyCompleted("Job already completed".to_string()))
|
||||
}
|
||||
},
|
||||
};
|
||||
if result.is_err() {
|
||||
for handle in tool_abort_handles.lock().unwrap().drain(..) {
|
||||
handle.abort();
|
||||
}
|
||||
cleanup_orphaned_tool_jobs(db, &job.id, &job.workspace_id, canceled_by.clone()).await;
|
||||
}
|
||||
cleanup_mcp_clients(mcp_clients).await;
|
||||
result
|
||||
}
|
||||
|
||||
/// OpenAI rejects a `prompt_cache_key` over 64 characters
|
||||
@@ -1231,12 +1231,14 @@ pub async fn run_agent(
|
||||
has_stream: &mut bool,
|
||||
has_websearch: bool,
|
||||
omit_output_from_conversation: bool,
|
||||
preserve_step_tags: bool,
|
||||
|
||||
// cancellation signal from parent
|
||||
cancel_rx: tokio::sync::watch::Receiver<bool>,
|
||||
|
||||
// abort handles for spawned tool tasks
|
||||
tool_abort_handles: ToolAbortHandles,
|
||||
job_completed_tx: crate::JobCompletedSender,
|
||||
) -> error::Result<Box<RawValue>> {
|
||||
let output_type = args.output_type.as_ref().unwrap_or(&OutputType::Text);
|
||||
let credentials = args.provider.to_provider_credentials(db).await?;
|
||||
@@ -2105,6 +2107,7 @@ pub async fn run_agent(
|
||||
stream_event_processor: stream_event_processor.as_ref(),
|
||||
flow_context: &mut flow_context,
|
||||
omit_output_from_conversation,
|
||||
preserve_step_tags,
|
||||
reasoning: if structured_output_first {
|
||||
None
|
||||
} else {
|
||||
@@ -2113,6 +2116,7 @@ pub async fn run_agent(
|
||||
previous_result: &previous_result,
|
||||
id_context: &id_context,
|
||||
tool_abort_handles: tool_abort_handles.clone(),
|
||||
job_completed_tx: job_completed_tx.clone(),
|
||||
};
|
||||
|
||||
let (tool_messages, tool_content, tool_used_structured_output) =
|
||||
@@ -3093,11 +3097,13 @@ async fn cleanup_orphaned_tool_jobs(
|
||||
)
|
||||
});
|
||||
|
||||
// Find direct child jobs still in v2_job_queue (agent tool jobs are always direct children)
|
||||
let orphaned_ids: Vec<Uuid> = match sqlx::query_scalar!(
|
||||
r#"SELECT j.id FROM v2_job j
|
||||
JOIN v2_job_queue q ON q.id = j.id
|
||||
WHERE j.parent_job = $1 AND j.workspace_id = $2"#,
|
||||
r#"WITH RECURSIVE descendants AS (
|
||||
SELECT id FROM v2_job WHERE parent_job = $1 AND workspace_id = $2
|
||||
UNION ALL
|
||||
SELECT j.id FROM v2_job j JOIN descendants d ON j.parent_job = d.id
|
||||
WHERE j.workspace_id = $2
|
||||
) SELECT d.id AS "id!" FROM descendants d JOIN v2_job_queue q ON q.id = d.id"#,
|
||||
parent_job_id,
|
||||
w_id,
|
||||
)
|
||||
|
||||
@@ -2969,7 +2969,7 @@ mod tests {
|
||||
assert!(
|
||||
attach.starts_with(&format!(
|
||||
"ATTACH 'sslmode=verify-full sslrootcert=''{}''",
|
||||
root
|
||||
root.replace('\\', "\\\\")
|
||||
)),
|
||||
"{attach}"
|
||||
);
|
||||
|
||||
@@ -828,7 +828,7 @@ pub async fn handle_python_job(
|
||||
for k, v in list(pre_args.items()):
|
||||
if v == '<function call>':
|
||||
del pre_args[k]
|
||||
kwargs = inner_script.preprocessor(**pre_args)
|
||||
kwargs = _await(inner_script.preprocessor(**pre_args))
|
||||
kwrags_json = res_to_json(kwargs, type(kwargs))
|
||||
with open("args.json", 'w', encoding="utf-8") as f:
|
||||
f.write(kwrags_json)"#
|
||||
@@ -969,6 +969,8 @@ result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.j
|
||||
def res_to_json(res, typ):
|
||||
{res_to_json_body}
|
||||
|
||||
{PY_AWAIT_HELPER}
|
||||
|
||||
try:
|
||||
{preprocessor}
|
||||
{spread}
|
||||
@@ -977,7 +979,7 @@ try:
|
||||
del args[k]
|
||||
if inner_script.{main_override} is None or not callable(inner_script.{main_override}):
|
||||
raise ValueError("{main_override} function is missing")
|
||||
res = inner_script.{main_override}(**args)
|
||||
res = _await(inner_script.{main_override}(**args))
|
||||
typ = type(res)
|
||||
if hasattr(res, '__iter__') and not isinstance(res, (str, dict, list, bytes, tuple, set, frozenset, range, memoryview, bytearray)) and typ.__name__ != 'DataFrame':
|
||||
for chunk in res:
|
||||
@@ -1513,6 +1515,8 @@ _fix=lambda s:s if 'Infinity' not in s and 'NaN' not in s and '\\u0000' not in s
|
||||
|
||||
def res_to_json(res, typ):
|
||||
{res_to_json_body}
|
||||
|
||||
{PY_AWAIT_HELPER}
|
||||
{functions}
|
||||
{registrations}
|
||||
|
||||
@@ -1546,11 +1550,11 @@ for line in sys.stdin:
|
||||
continue
|
||||
kwargs = json.loads(args_json, strict=False)
|
||||
pre_args = entry['pre_transform'](kwargs)
|
||||
preprocessed = mod.preprocessor(**pre_args)
|
||||
preprocessed = _await(mod.preprocessor(**pre_args))
|
||||
preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '')
|
||||
sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n")
|
||||
main_args = entry['transform'](preprocessed if preprocessed else {{}})
|
||||
res = mod.main(**main_args)
|
||||
res = _await(mod.main(**main_args))
|
||||
typ = type(res)
|
||||
res_json = res_to_json(res, typ)
|
||||
sys.stdout.write("wm_res[success]:" + res_json + "\n")
|
||||
@@ -1574,11 +1578,11 @@ for line in sys.stdin:
|
||||
continue
|
||||
kwargs = json.loads(args_json, strict=False)
|
||||
pre_args = entry['pre_transform'](kwargs)
|
||||
preprocessed = mod.preprocessor(**pre_args)
|
||||
preprocessed = _await(mod.preprocessor(**pre_args))
|
||||
preprocessed_json = json.dumps(preprocessed, separators=(',', ':'), default=str).replace('\n', '')
|
||||
sys.stdout.write("wm_res[preprocessed_args]:" + preprocessed_json + "\n")
|
||||
main_args = entry['transform'](preprocessed if preprocessed else {{}})
|
||||
res = mod.main(**main_args)
|
||||
res = _await(mod.main(**main_args))
|
||||
typ = type(res)
|
||||
res_json = res_to_json(res, typ)
|
||||
sys.stdout.write("wm_res[success]:" + res_json + "\n")
|
||||
@@ -1596,7 +1600,7 @@ for line in sys.stdin:
|
||||
entry = next(iter(scripts.values()))
|
||||
kwargs = json.loads(args_json, strict=False)
|
||||
args = entry['transform'](kwargs)
|
||||
res = entry['mod'].main(**args)
|
||||
res = _await(entry['mod'].main(**args))
|
||||
typ = type(res)
|
||||
res_json = res_to_json(res, typ)
|
||||
sys.stdout.write("wm_res[success]:" + res_json + "\n")
|
||||
@@ -1624,7 +1628,7 @@ for line in sys.stdin:
|
||||
|
||||
kwargs = json.loads(args_json, strict=False)
|
||||
args = entry['transform'](kwargs)
|
||||
res = entry['mod'].main(**args)
|
||||
res = _await(entry['mod'].main(**args))
|
||||
typ = type(res)
|
||||
res_json = res_to_json(res, typ)
|
||||
sys.stdout.write("wm_res[success]:" + res_json + "\n")
|
||||
@@ -3330,6 +3334,31 @@ This is not normal behavior, please make sure all workers have enough memory.\n
|
||||
};
|
||||
}
|
||||
|
||||
/// Python helper running what an `async def` main or preprocessor returns.
|
||||
/// One loop per process: a dedicated worker's module-level async clients and
|
||||
/// locks are bound to the loop they were first used on, so `asyncio.run`'s
|
||||
/// loop-per-call would break them from the second job on. A loop the script
|
||||
/// set while importing (`asyncio.set_event_loop`) is reused. asyncio is
|
||||
/// imported lazily so sync jobs don't pay its import time.
|
||||
const PY_AWAIT_HELPER: &str = r#"_loop = None
|
||||
def _await(r):
|
||||
global _loop
|
||||
if not hasattr(r, '__await__'):
|
||||
return r
|
||||
import asyncio
|
||||
if _loop is None or _loop.is_closed():
|
||||
import warnings
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', DeprecationWarning)
|
||||
_loop = asyncio.get_event_loop()
|
||||
if _loop.is_closed():
|
||||
raise RuntimeError
|
||||
except RuntimeError:
|
||||
_loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(_loop)
|
||||
return _loop.run_until_complete(r)"#;
|
||||
|
||||
/// Python function body for `res_to_json(res, typ)`.
|
||||
/// Handles DataFrame, bytes, dict coercion + JSON serialization.
|
||||
fn python_res_to_json_body(postprocessor: &str) -> String {
|
||||
|
||||
@@ -4990,6 +4990,7 @@ pub async fn handle_queued_job(
|
||||
hostname,
|
||||
killpill_rx,
|
||||
&mut has_stream,
|
||||
job_completed_tx.clone(),
|
||||
))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -3239,7 +3239,7 @@ struct PushNextFlowJobRec {
|
||||
/// - when the flow opts into `preserve_step_tags` and the child declares its own non-empty tag,
|
||||
/// that tag is honored instead of being overridden by the flow tag;
|
||||
/// - otherwise the child inherits the parent flow job's tag.
|
||||
fn resolve_flow_step_tag(
|
||||
pub(crate) fn resolve_flow_step_tag(
|
||||
is_preprocessor_step: bool,
|
||||
flow_tag: &str,
|
||||
workspace_id: &str,
|
||||
|
||||
@@ -77,6 +77,7 @@ import {
|
||||
import {
|
||||
getEffectiveSettings,
|
||||
inferWsNameFromProfile,
|
||||
inferWsNameFromForkBranch,
|
||||
resolveWsNameForConfigFromFlags,
|
||||
mergeConfigWithConfigFile,
|
||||
parseSyncBehavior,
|
||||
@@ -3752,9 +3753,12 @@ export async function pull(
|
||||
}
|
||||
}
|
||||
|
||||
// If wsNameForConfig wasn't set from flags, infer from the resolved profile
|
||||
// If wsNameForConfig wasn't set from flags, infer from the resolved profile,
|
||||
// then (for a fork) from the parent's entry
|
||||
if (!wsNameForConfig) {
|
||||
wsNameForConfig = inferWsNameFromProfile(opts, workspace);
|
||||
wsNameForConfig =
|
||||
inferWsNameFromProfile(opts, workspace) ??
|
||||
inferWsNameFromForkBranch(opts, workspace.workspaceId);
|
||||
}
|
||||
|
||||
// Resolve effective sync options with workspace awareness
|
||||
@@ -4802,9 +4806,12 @@ export async function push(
|
||||
const workspace = await resolveWorkspace(opts, wsNameForConfig);
|
||||
await requireLogin(opts);
|
||||
|
||||
// If wsNameForConfig wasn't set from flags, infer from the resolved profile
|
||||
// If wsNameForConfig wasn't set from flags, infer from the resolved profile,
|
||||
// then (for a fork) from the parent's entry
|
||||
if (!wsNameForConfig) {
|
||||
wsNameForConfig = inferWsNameFromProfile(opts, workspace);
|
||||
wsNameForConfig =
|
||||
inferWsNameFromProfile(opts, workspace) ??
|
||||
inferWsNameFromForkBranch(opts, workspace.workspaceId);
|
||||
}
|
||||
|
||||
// Resolve effective sync options with workspace awareness
|
||||
|
||||
@@ -5,6 +5,7 @@ import { stringify as yamlStringify } from "yaml";
|
||||
import {
|
||||
getCurrentGitBranch,
|
||||
getOriginalBranchForWorkspaceForks,
|
||||
getWorkspaceIdForWorkspaceForkFromBranchName,
|
||||
isGitRepository,
|
||||
} from "../utils/git.ts";
|
||||
import { join, dirname, resolve, relative } from "node:path";
|
||||
@@ -725,6 +726,30 @@ export function inferWsNameFromProfile(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The workspace config entry a fork inherits from its parent: the one bound to
|
||||
* the base branch of the checked-out `wm-fork/<base>/<id>` branch, when
|
||||
* `workspaceId` is the fork that branch names. A fork id never has an entry of
|
||||
* its own, so without this its specific items and file suffix fall back to
|
||||
* the common/base files and clobber what the fork cloned from its parent.
|
||||
*/
|
||||
export function inferWsNameFromForkBranch(
|
||||
opts: SyncOptions,
|
||||
workspaceId: string
|
||||
): string | undefined {
|
||||
if (!opts.workspaces || !isGitRepository()) return undefined;
|
||||
const branch = getCurrentGitBranch();
|
||||
const baseBranch = getOriginalBranchForWorkspaceForks(branch);
|
||||
if (
|
||||
!branch ||
|
||||
!baseBranch ||
|
||||
getWorkspaceIdForWorkspaceForkFromBranchName(branch) !== workspaceId
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return findWorkspaceByGitBranch(opts.workspaces, baseBranch)?.[0];
|
||||
}
|
||||
|
||||
/**
|
||||
* `syncBehavior` as the workspace being pushed to sees it. The top level alone
|
||||
* misses a `workspaces.<name>.overrides.syncBehavior`, which is where a repo
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { execSync } from "node:child_process";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { inferWsNameFromForkBranch, type SyncOptions } from "../src/core/conf.ts";
|
||||
|
||||
// A fork id never has a wmill.yaml entry of its own. Unless it resolves to its
|
||||
// parent's entry, a sync on the fork branch pushes the base files over the
|
||||
// workspace-specific values the fork cloned from its parent.
|
||||
test("a fork branch resolves to its parent's workspace entry", async () => {
|
||||
const repoDir = await mkdtemp(path.join(os.tmpdir(), "wmill_fork_specific_"));
|
||||
const originalCwd = process.cwd();
|
||||
try {
|
||||
execSync(
|
||||
`git init -q -b wm-fork/dev-branch/t1 && git -c user.email=t@t -c user.name=t commit -q --allow-empty -m init`,
|
||||
{ cwd: repoDir },
|
||||
);
|
||||
process.chdir(repoDir);
|
||||
|
||||
const config: SyncOptions = {
|
||||
workspaces: {
|
||||
dev: {
|
||||
baseUrl: "http://localhost:8000/",
|
||||
workspaceId: "dev-ws",
|
||||
gitBranch: "dev-branch",
|
||||
specificItems: { folders: ["f/team"] },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
expect(inferWsNameFromForkBranch(config, "wm-fork-t1")).toEqual("dev");
|
||||
// Only the fork the branch names inherits the parent's entry.
|
||||
expect(inferWsNameFromForkBranch(config, "other-ws")).toBeUndefined();
|
||||
} finally {
|
||||
process.chdir(originalCwd);
|
||||
await rm(repoDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
@@ -455,6 +455,12 @@ schema:
|
||||
// SCRIPT WITH MODULES PREVIEW TESTS
|
||||
// =============================================================================
|
||||
|
||||
// Pinned: a release commit runs these tests while the same release publishes a new
|
||||
// windmill-client, and the worker resolving that version before npm serves it fails the job.
|
||||
// These are the only CLI tests running task/taskScript/workflow on a real client: bump the pin
|
||||
// to a published release whenever the WAC client surface changes.
|
||||
const WINDMILL_CLIENT = "windmill-client@1.819.0";
|
||||
|
||||
test("script preview: script with modules (taskScript pattern)", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, { defaultTs: "bun" });
|
||||
@@ -463,7 +469,7 @@ test("script preview: script with modules (taskScript pattern)", async () => {
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/test/wac_script.ts",
|
||||
`import { task, taskScript, workflow } from "windmill-client";
|
||||
`import { task, taskScript, workflow } from "${WINDMILL_CLIENT}";
|
||||
|
||||
const helper = taskScript("./helper.ts");
|
||||
|
||||
@@ -512,7 +518,7 @@ test("script preview: script with modules (folder layout)", async () => {
|
||||
// Entry point script
|
||||
await writeFile(
|
||||
`${modDir}/script.ts`,
|
||||
`import { task, taskScript, workflow } from "windmill-client";
|
||||
`import { task, taskScript, workflow } from "${WINDMILL_CLIENT}";
|
||||
|
||||
const helper = taskScript("./helper.ts");
|
||||
|
||||
|
||||
@@ -61,8 +61,7 @@ root.render(<App/>);
|
||||
const PACKAGE_JSON = `{
|
||||
"dependencies": {
|
||||
"react": "19.0.0",
|
||||
"react-dom": "19.0.0",
|
||||
"windmill-client": "^1"
|
||||
"react-dom": "19.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/react-dom": "^19.0.0",
|
||||
|
||||
@@ -123,7 +123,13 @@
|
||||
</script>
|
||||
|
||||
<InitializeComponent {id} />
|
||||
<div class="w-full h-full overflow-auto {customComponent?.name ?? 'no_name'}">
|
||||
<!-- The host stays mounted so the renderer can attach, but the user's code draws regardless of
|
||||
render: hide it, or a not-yet-rendered component lands in its parent's layout flow. Inline so
|
||||
CSS the bundle puts on its root class cannot override it. -->
|
||||
<div
|
||||
class="w-full h-full overflow-auto {customComponent?.name ?? 'no_name'}"
|
||||
style:display={render ? undefined : 'none'}
|
||||
>
|
||||
<RunnableWrapper {outputs} render autoRefresh {componentInput} {id} bind:result>
|
||||
{#if !loaded && render}
|
||||
<Loader2 class="animate-spin" />
|
||||
|
||||
@@ -208,9 +208,12 @@
|
||||
// Keyed by call id: a bare flag would carry one card's collapse onto the next message
|
||||
// reusing this instance. Open by default, since the run is what was asked for —
|
||||
// except for an inspection, which is usually a step in the reasoning rather than
|
||||
// the answer, and which pays a fetch for being opened.
|
||||
// the answer, and which pays a fetch for being opened. A run cancelled before it started
|
||||
// is closed too: the row already says Cancelled, and there is no run to show.
|
||||
let toggled = $state<{ id: string; open: boolean } | undefined>(undefined)
|
||||
const expanded = $derived(toggled?.id === message.tool_call_id ? toggled.open : !inspected)
|
||||
const expanded = $derived(
|
||||
toggled?.id === message.tool_call_id ? toggled.open : !inspected && (ran || !canceled)
|
||||
)
|
||||
|
||||
$effect(() => {
|
||||
const target = inspected
|
||||
|
||||
@@ -22,6 +22,8 @@ import {
|
||||
} from './modelConfig'
|
||||
import {
|
||||
applyReasoningToConfig,
|
||||
completionsRejectsToolsWithReasoning,
|
||||
explicitOffToken,
|
||||
requestsReasoning,
|
||||
stripLegacyThinkingSuffix,
|
||||
type ReasoningEffort
|
||||
@@ -69,6 +71,9 @@ interface AIProviderDetails {
|
||||
// the frontier model. The gpt-5 family is deprecated (retires 2026-12-11) but
|
||||
// still served, so it stays in the list below the 5.6 models.
|
||||
const OPENAI_MODELS = [
|
||||
'gpt-6-sol',
|
||||
'gpt-6-astra',
|
||||
'gpt-6-luna',
|
||||
'gpt-5.6-terra',
|
||||
'gpt-5.6-sol',
|
||||
'gpt-5.6-luna',
|
||||
@@ -87,7 +92,14 @@ export const AI_PROVIDERS: Record<AIProvider, AIProviderDetails> = {
|
||||
},
|
||||
anthropic: {
|
||||
label: 'Anthropic',
|
||||
defaultModels: ['claude-sonnet-5', 'claude-opus-5', 'claude-opus-4-8', 'claude-haiku-4-5']
|
||||
defaultModels: [
|
||||
'claude-sonnet-5-5',
|
||||
'claude-opus-5-5',
|
||||
'claude-sonnet-5',
|
||||
'claude-opus-5',
|
||||
'claude-opus-4-8',
|
||||
'claude-haiku-4-5'
|
||||
]
|
||||
},
|
||||
googleai: {
|
||||
label: 'Google AI',
|
||||
@@ -1160,6 +1172,12 @@ export async function getCompletion(
|
||||
|
||||
// Use Completions API for other providers
|
||||
const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient()
|
||||
// gpt-5.5+ refuse function tools here unless reasoning is off, so the Responses API
|
||||
// fallback turns it off where the model can rather than failing the turn.
|
||||
const reasoningEffort =
|
||||
tools?.length && completionsRejectsToolsWithReasoning(provider, modelProvider.model)
|
||||
? (explicitOffToken(provider, modelProvider.model) ?? options?.reasoningEffort)
|
||||
: options?.reasoningEffort
|
||||
const completionConfig = applyReasoningToConfig(
|
||||
config.stream && STREAM_USAGE_PROVIDERS.has(provider)
|
||||
? {
|
||||
@@ -1175,7 +1193,7 @@ export async function getCompletion(
|
||||
}
|
||||
: config,
|
||||
provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions',
|
||||
options?.reasoningEffort
|
||||
reasoningEffort
|
||||
)
|
||||
const completion = client.chat.completions.create(completionConfig, {
|
||||
signal: abortController.signal,
|
||||
|
||||
@@ -21,6 +21,13 @@ describe('workspace context window overrides', () => {
|
||||
expect(getConfiguredModelContextWindow('openai', 'qwen-local', overrides)).toBeUndefined()
|
||||
expect(getEffectiveModelContextWindow('openai', 'qwen-local', overrides)).toBe(128_000)
|
||||
})
|
||||
|
||||
it('knows the gpt-6 and Claude 5.5 windows, so they do not compact at the assumed one', () => {
|
||||
expect(getConfiguredModelContextWindow('openai', 'gpt-6-sol', undefined)).toBe(1_050_000)
|
||||
expect(getConfiguredModelContextWindow('anthropic', 'claude-sonnet-5-5', undefined)).toBe(
|
||||
1_000_000
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('usesAnthropicMessagesApi', () => {
|
||||
|
||||
@@ -58,7 +58,7 @@ export function usesOpenRouterPromptCaching(provider: AIProvider, model: string)
|
||||
// so it does not catch unrelated ids like Mistral's "open-mistral-*" or "optimus-*".
|
||||
export function requiresMaxCompletionTokens(model: string) {
|
||||
const baseModel = parseModelId(model).base
|
||||
return baseModel.startsWith('gpt-5') || /^o\d/.test(baseModel)
|
||||
return Number(/^gpt-(\d)(?:[.-]|$)/.exec(baseModel)?.[1] ?? 0) >= 5 || /^o\d/.test(baseModel)
|
||||
}
|
||||
|
||||
// Context windows of the models we know, most specific entry first — the first
|
||||
@@ -87,6 +87,7 @@ const MODEL_CONTEXT_WINDOWS: [name: string, contextWindow: number][] = [
|
||||
['claude', 200_000],
|
||||
// OpenAI — gpt-5 covers the base family (-mini / -nano) and the 5.1/5.2
|
||||
// revisions, all 400K; 5.4/5.5 moved to 1M and 5.6 to 1.05M
|
||||
['gpt-6', 1_050_000],
|
||||
['gpt-5.6', 1_050_000],
|
||||
['gpt-5.5', 1_000_000],
|
||||
['gpt-5.4', 1_000_000],
|
||||
@@ -140,6 +141,7 @@ const MODEL_MAX_OUTPUT_TOKENS: [name: string, maxOutputTokens: number][] = [
|
||||
['claude-fable', 64_000],
|
||||
['claude-mythos', 64_000],
|
||||
// OpenAI
|
||||
['gpt-6', 128_000],
|
||||
['gpt-5', 128_000],
|
||||
['gpt-4.1', 32_768],
|
||||
['gpt-4o', 16_384],
|
||||
|
||||
@@ -35,6 +35,16 @@ describe('resolveModelPrice', () => {
|
||||
expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('prices Claude 5.5 flat and leaves the tiered gpt-6 unpriced', () => {
|
||||
expect(resolveModelPrice('anthropic', 'claude-opus-5-5', undefined)?.price).toMatchObject({
|
||||
input: 4,
|
||||
output: 20,
|
||||
cacheRead: 0.2
|
||||
})
|
||||
expect(resolveModelPrice('anthropic', 'claude-sonnet-5-5', undefined)?.price.input).toBe(2)
|
||||
expect(resolveModelPrice('openai', 'gpt-6-sol', undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('reports an unknown model as unpriced rather than guessing', () => {
|
||||
expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
@@ -69,6 +69,8 @@ const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [
|
||||
// fallback sits below the explicit entries rather than covering them.
|
||||
['claude-fable-5', { input: 10, output: 50 }],
|
||||
['claude-mythos-5', { input: 10, output: 50 }],
|
||||
['claude-opus-5-5', { input: 4, output: 20, cacheRead: 0.2 }],
|
||||
['claude-sonnet-5-5', { input: 2, output: 10, cacheRead: 0.2 }],
|
||||
['claude-opus-5', { input: 5, output: 25 }],
|
||||
['claude-opus-4-8', { input: 5, output: 25 }],
|
||||
['claude-opus-4-7', { input: 5, output: 25 }],
|
||||
@@ -98,6 +100,9 @@ const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [
|
||||
// Revisions past gpt-5 are priced separately by OpenAI and are not tracked here.
|
||||
// The matcher's revision guard already keeps them off the family rate; these
|
||||
// entries stay so a revision the guard admits still resolves to no rate.
|
||||
// gpt-6 bills the whole request at a higher rate above 272K input tokens, which a
|
||||
// per-model rate cannot express.
|
||||
['gpt-6', null],
|
||||
['gpt-5.6', null],
|
||||
['gpt-5.5', null],
|
||||
['gpt-5.4', null],
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
[
|
||||
{
|
||||
"provider": "anthropic",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"canDisable": false,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{
|
||||
"provider": "anthropic",
|
||||
"model": "claude-opus-5-5",
|
||||
"canDisable": false,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{
|
||||
"provider": "anthropic",
|
||||
"model": "claude-fable-5",
|
||||
"canDisable": false,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{
|
||||
"provider": "aws_bedrock",
|
||||
"model": "global.anthropic.claude-opus-5-5-v1:0",
|
||||
"canDisable": false,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{
|
||||
"provider": "anthropic",
|
||||
"model": "claude-sonnet-5",
|
||||
"canDisable": true,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{
|
||||
"provider": "anthropic",
|
||||
"model": "claude-opus-5-20260101",
|
||||
"canDisable": true,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{
|
||||
"provider": "anthropic",
|
||||
"model": "claude-opus-4-8",
|
||||
"canDisable": true,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "gpt-6-astra",
|
||||
"canDisable": false,
|
||||
"completionsToolsNeedOff": true
|
||||
},
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "gpt-6-sol",
|
||||
"canDisable": true,
|
||||
"completionsToolsNeedOff": true
|
||||
},
|
||||
{
|
||||
"provider": "azure_openai",
|
||||
"model": "gpt-6-luna",
|
||||
"canDisable": true,
|
||||
"completionsToolsNeedOff": true
|
||||
},
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "gpt-5.6-sol",
|
||||
"canDisable": true,
|
||||
"completionsToolsNeedOff": true
|
||||
},
|
||||
{ "provider": "openai", "model": "gpt-5.5", "canDisable": true, "completionsToolsNeedOff": true },
|
||||
{ "provider": "openai", "model": "gpt-5.7", "canDisable": true, "completionsToolsNeedOff": true },
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "gpt-5.1",
|
||||
"canDisable": true,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{ "provider": "openai", "model": "gpt-5", "canDisable": false, "completionsToolsNeedOff": false },
|
||||
{
|
||||
"provider": "openai",
|
||||
"model": "gpt-5-mini",
|
||||
"canDisable": false,
|
||||
"completionsToolsNeedOff": false
|
||||
},
|
||||
{ "provider": "openai", "model": "o3", "canDisable": false, "completionsToolsNeedOff": false }
|
||||
]
|
||||
@@ -1,6 +1,8 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
applyReasoningToConfig,
|
||||
completionsRejectsToolsWithReasoning,
|
||||
explicitOffToken,
|
||||
getReasoningCapability,
|
||||
REASONING_OFF,
|
||||
resolveEffectiveReasoning,
|
||||
@@ -8,6 +10,8 @@ import {
|
||||
stripLegacyThinkingSuffix,
|
||||
supportsReasoning
|
||||
} from './reasoningRegistry'
|
||||
import type { AIProvider } from '$lib/gen'
|
||||
import parity from './reasoningParity.json'
|
||||
|
||||
describe('stripLegacyThinkingSuffix', () => {
|
||||
it('removes the deprecated /thinking suffix', () => {
|
||||
@@ -261,6 +265,42 @@ describe('supportsReasoning (static registry)', () => {
|
||||
expect(getReasoningCapability('openrouter', 'x-ai/grok-4').canDisable).toBe(false)
|
||||
expect(getReasoningCapability('openrouter', 'deepseek/deepseek-r1').canDisable).toBe(false)
|
||||
})
|
||||
it('never sends a disable the 5.5 point releases and gpt-6-astra reject', () => {
|
||||
// Live-verified: Claude 5.5 rejects `thinking: disabled`, gpt-6-astra rejects `none`.
|
||||
for (const [provider, model] of [
|
||||
['anthropic', 'claude-sonnet-5-5'],
|
||||
['anthropic', 'claude-opus-5-5'],
|
||||
['aws_bedrock', 'global.anthropic.claude-opus-5-5-v1:0'],
|
||||
['openai', 'gpt-6-astra']
|
||||
] as const) {
|
||||
expect(getReasoningCapability(provider, model).canDisable, model).toBe(false)
|
||||
expect(explicitOffToken(provider, model), model).toBeUndefined()
|
||||
}
|
||||
expect(getReasoningCapability('openrouter', 'anthropic/claude-sonnet-5.5').canDisable).toBe(
|
||||
false
|
||||
)
|
||||
expect(explicitOffToken('openrouter', 'anthropic/claude-sonnet-5.5')).toBeUndefined()
|
||||
// A dated Claude 5 id is not a point release.
|
||||
expect(explicitOffToken('anthropic', 'claude-sonnet-5-20260101')).toBe('none')
|
||||
expect(explicitOffToken('openai', 'gpt-6-sol')).toBe('none')
|
||||
expect(getReasoningCapability('openai', 'gpt-6-luna')).toMatchObject({
|
||||
supported: true,
|
||||
canDisable: true,
|
||||
levels: ['low', 'medium', 'high', 'xhigh', 'max']
|
||||
})
|
||||
})
|
||||
it("reads Azure's gpt-35-turbo as gpt-3.5, not a gpt-5+ reasoning model", () => {
|
||||
expect(supportsReasoning('azure_openai', 'gpt-35-turbo')).toBe(false)
|
||||
expect(supportsReasoning('openai', 'gpt-35-turbo-16k')).toBe(false)
|
||||
})
|
||||
it('finds the models that refuse function tools with reasoning on Chat Completions', () => {
|
||||
for (const model of ['gpt-5.5', 'gpt-5.6-sol', 'gpt-6-astra']) {
|
||||
expect(completionsRejectsToolsWithReasoning('openai', model), model).toBe(true)
|
||||
}
|
||||
for (const model of ['gpt-5', 'gpt-5.1', 'gpt-35-turbo', 'o3']) {
|
||||
expect(completionsRejectsToolsWithReasoning('azure_openai', model), model).toBe(false)
|
||||
}
|
||||
})
|
||||
it('forwards an explicit off as effort none through OpenRouter', () => {
|
||||
expect(
|
||||
resolveRequestReasoning({
|
||||
@@ -353,6 +393,23 @@ describe('Azure AI Foundry reasoning follows the model family', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('backend parity', () => {
|
||||
// windmill-ai's `providers/mod.rs` test reads the same file. The backend rules see the
|
||||
// model id alone, so the file holds only rows whose answer doesn't depend on the
|
||||
// provider: a Bedrock- or Gemini-Pro-specific row belongs in the tests above.
|
||||
it.each(parity)(
|
||||
'$provider $model',
|
||||
({ provider, model, canDisable, completionsToolsNeedOff }) => {
|
||||
const capability = getReasoningCapability(provider as AIProvider, model)
|
||||
expect(capability.supported).toBe(true)
|
||||
expect(capability.canDisable).toBe(canDisable)
|
||||
expect(completionsRejectsToolsWithReasoning(provider as AIProvider, model)).toBe(
|
||||
completionsToolsNeedOff
|
||||
)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
describe('resolveEffectiveReasoning', () => {
|
||||
it('defaults capable models to high when unset', () => {
|
||||
expect(resolveEffectiveReasoning({ provider: 'anthropic', model: 'claude-sonnet-4-6' })).toBe(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { AIProvider, AIProviderModel } from '$lib/gen'
|
||||
import { parseModelId, usesAnthropicMessagesApi } from './modelConfig'
|
||||
import { usesAnthropicMessagesApi } from './modelConfig'
|
||||
|
||||
/**
|
||||
* Reasoning effort is provider/model-specific. We never normalize a single
|
||||
@@ -33,11 +33,6 @@ export function stripLegacyThinkingSuffix(model: string): string {
|
||||
: model
|
||||
}
|
||||
|
||||
/** Bare model id without any provider/gateway prefix (e.g. OpenRouter's `openai/o3`). */
|
||||
function baseModelId(model: string): string {
|
||||
return parseModelId(model).base
|
||||
}
|
||||
|
||||
/**
|
||||
* Azure AI Foundry hosts multiple model families under one provider, so reasoning
|
||||
* support follows the underlying model rather than the provider: Claude deployments
|
||||
@@ -53,172 +48,218 @@ function reasoningProviderFamily(provider: AIProvider, model: string): AIProvide
|
||||
}
|
||||
|
||||
/**
|
||||
* Suggested effort levels per provider, sourced from each provider SDK's own
|
||||
* vocabulary.
|
||||
* Sentinel sent for the deepseek off case. It never reaches the wire as an
|
||||
* effort: the 'deepseek' branch of `applyReasoningToConfig` translates it to
|
||||
* the provider's `thinking: {type: "disabled"}` param (`reasoning_effort:
|
||||
* "none"` is rejected by their API).
|
||||
*/
|
||||
const PROVIDER_REASONING_LEVELS: Partial<Record<AIProvider, ReasoningEffort[]>> = {
|
||||
// DeepSeek accepts the full five-token vocabulary but only two levels are
|
||||
// real: low/medium are server-mapped to high and xhigh to max — offering
|
||||
// them would be a no-op knob.
|
||||
deepseek: ['high', 'max'],
|
||||
// Mistral's only effort token besides the 'none' disable is 'high'
|
||||
// (anything else is rejected), so the knob is effectively on/off.
|
||||
mistral: ['high']
|
||||
}
|
||||
export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none'
|
||||
|
||||
/**
|
||||
* OpenRouter validates effort against its own vocabulary
|
||||
* (minimal..xhigh + none) and translates per underlying provider, so the
|
||||
* real ladder depends on the model family: Anthropic gets all five as
|
||||
* distinct budget ratios (minimal 10% .. xhigh 95% of max_tokens); Gemini
|
||||
* maps to thinkingLevel with xhigh clamped to high (a no-op vs high);
|
||||
* OpenAI gets the token passed through verbatim, so the per-model OpenAI
|
||||
* scoping applies; DeepSeek server-maps low/medium to high and xhigh to max.
|
||||
* Sentinel for the Anthropic off case. Like the DeepSeek one it never reaches
|
||||
* the wire as an effort: the 'anthropic' branch of `applyReasoningToConfig`
|
||||
* translates it to `thinking: {type: "disabled"}`, the only off that Opus and
|
||||
* Sonnet 5 respect.
|
||||
*/
|
||||
function openrouterReasoningLevels(model: string): ReasoningEffort[] {
|
||||
const m = model.toLowerCase()
|
||||
const base = baseModelId(model)
|
||||
if (/claude-(opus|sonnet)-(4|5)/.test(m)) {
|
||||
return ['minimal', 'low', 'medium', 'high', 'xhigh']
|
||||
}
|
||||
if (m.includes('gemini-')) {
|
||||
return geminiReasoningLevels(m)
|
||||
}
|
||||
if (base.startsWith('gpt-5') || /^o\d/.test(base)) {
|
||||
return openaiReasoningLevels(base)
|
||||
}
|
||||
if (m.includes('deepseek-v4')) {
|
||||
return ['high', 'xhigh']
|
||||
}
|
||||
return ['low', 'medium', 'high']
|
||||
}
|
||||
export const ANTHROPIC_OFF_SENTINEL: ReasoningEffort = 'none'
|
||||
|
||||
/**
|
||||
* OpenAI's effort vocabulary is model-dependent: `minimal` exists on gpt-5 but
|
||||
* not on gpt-5.1+, `xhigh` arrived on gpt-5.5 and `max` on gpt-5.6; o-series
|
||||
* take low/medium/high. An unsupported level is rejected, so scope the list to
|
||||
* the model. (`none` is the disable token, handled by `explicitOffToken`.)
|
||||
* What the registry knows about one group of models. The rows below are matched in
|
||||
* order against the lowercased model id, first match wins, so a narrower row goes above
|
||||
* the row it overrides. A model no row matches does not reason, as far as we know.
|
||||
*/
|
||||
function openaiReasoningLevels(model: string): ReasoningEffort[] {
|
||||
const base = baseModelId(model)
|
||||
if (/^gpt-5\.6/.test(base)) {
|
||||
return ['low', 'medium', 'high', 'xhigh', 'max']
|
||||
}
|
||||
if (/^gpt-5\.5/.test(base)) {
|
||||
return ['low', 'medium', 'high', 'xhigh']
|
||||
}
|
||||
if (/^gpt-5\./.test(base)) {
|
||||
return ['low', 'medium', 'high']
|
||||
}
|
||||
if (/^gpt-5/.test(base)) {
|
||||
return ['minimal', 'low', 'medium', 'high']
|
||||
}
|
||||
return ['low', 'medium', 'high']
|
||||
type ReasoningRule = {
|
||||
match: RegExp
|
||||
/** The levels the UI offers. An unsupported level is a 400, so this is per model. */
|
||||
levels: readonly ReasoningEffort[]
|
||||
/**
|
||||
* Whether "off" really stops the model reasoning. When false the UI offers no off:
|
||||
* the provider would reject it, or coerce it to the lowest level.
|
||||
*/
|
||||
canDisable: boolean
|
||||
/**
|
||||
* The effort to send for off on a model that reasons when the field is omitted.
|
||||
* Unset where omission is already off. Always `'none'`: `requestsReasoning` reads
|
||||
* that as off, and each wire format translates it (see `applyReasoningToConfig`).
|
||||
*/
|
||||
offToken?: ReasoningEffort
|
||||
/**
|
||||
* gpt-5.5 and later refuse function tools on Chat Completions while they reason,
|
||||
* even with the effort omitted (live-verified); the Responses API has no such limit.
|
||||
*/
|
||||
completionsToolsNeedOff?: boolean
|
||||
}
|
||||
|
||||
const LOW_TO_HIGH = ['low', 'medium', 'high']
|
||||
const LOW_TO_XHIGH = ['low', 'medium', 'high', 'xhigh']
|
||||
const LOW_TO_MAX = ['low', 'medium', 'high', 'xhigh', 'max']
|
||||
const MINIMAL_TO_HIGH = ['minimal', 'low', 'medium', 'high']
|
||||
|
||||
/**
|
||||
* Gemini's level ladder is model-dependent: Gemini 3+ Flash / Flash-Lite accept
|
||||
* `minimal`, while 3.x Pro does not (and cannot disable thinking). Gemini 2.5
|
||||
* uses numeric budgets — the proxy maps the three tiers to budget values, so
|
||||
* `minimal` is not offered there.
|
||||
* Claude models whose thinking cannot be turned off: an explicit disable 400s. Fable,
|
||||
* Mythos, and the 5.x point releases (Sonnet 5.5, Opus 5.5), whose lowest setting is
|
||||
* adaptive thinking at `low` (live-verified). The version match stops at one digit so a
|
||||
* dated id (`claude-sonnet-5-20260101`) stays Sonnet 5.
|
||||
*/
|
||||
function geminiReasoningLevels(model: string): ReasoningEffort[] {
|
||||
const m = model.toLowerCase()
|
||||
const isGemini3Plus = !m.includes('gemini-2.5')
|
||||
if (isGemini3Plus && (m.includes('flash') || m.includes('lite'))) {
|
||||
return ['minimal', 'low', 'medium', 'high']
|
||||
}
|
||||
return ['low', 'medium', 'high']
|
||||
const CLAUDE_ALWAYS_THINKING = /fable|mythos|claude-(opus|sonnet)-5[-.][1-9](?!\d)/
|
||||
|
||||
// Anthropic ids are matched anywhere in the id: Bedrock prefixes them
|
||||
// (`us.anthropic.claude-opus-4-6-v1`). Opus 4.5 and older reject adaptive thinking.
|
||||
const ANTHROPIC_RULES: ReasoningRule[] = [
|
||||
{ match: CLAUDE_ALWAYS_THINKING, levels: LOW_TO_MAX, canDisable: false },
|
||||
// The 5 family thinks when the field is absent, so off is an explicit disable.
|
||||
{
|
||||
match: /claude-(opus|sonnet)-5/,
|
||||
levels: LOW_TO_MAX,
|
||||
canDisable: true,
|
||||
offToken: ANTHROPIC_OFF_SENTINEL
|
||||
},
|
||||
// 4.6-4.8 only think when asked, so omission is already off.
|
||||
{ match: /claude-opus-4-[78]/, levels: LOW_TO_MAX, canDisable: true },
|
||||
{ match: /claude-(opus|sonnet)-4-6/, levels: ['low', 'medium', 'high', 'max'], canDisable: true }
|
||||
]
|
||||
|
||||
const BEDROCK_RULES: ReasoningRule[] = [
|
||||
ANTHROPIC_RULES[0],
|
||||
// AWS documents Sonnet 5 on Bedrock as always thinking, where the native API
|
||||
// accepts a disable for it.
|
||||
{ match: /claude-sonnet-5/, levels: LOW_TO_MAX, canDisable: false },
|
||||
...ANTHROPIC_RULES.slice(1)
|
||||
]
|
||||
|
||||
// Anchored at the start or after a gateway's `vendor/`, and the major is one digit:
|
||||
// Azure names gpt-3.5 `gpt-35-turbo`. `minimal` exists on gpt-5 only, `xhigh` from
|
||||
// gpt-5.5, `max` from gpt-5.6.
|
||||
const OPENAI_RULES: ReasoningRule[] = [
|
||||
// Live-verified: astra takes low..max only, where sol and luna also take `none`.
|
||||
{
|
||||
match: /(?:^|\/)gpt-6-astra/,
|
||||
levels: LOW_TO_MAX,
|
||||
canDisable: false,
|
||||
completionsToolsNeedOff: true
|
||||
},
|
||||
{
|
||||
match: /(?:^|\/)gpt-[6-9](?:[.:-]|$)/,
|
||||
levels: LOW_TO_MAX,
|
||||
canDisable: true,
|
||||
offToken: 'none',
|
||||
completionsToolsNeedOff: true
|
||||
},
|
||||
{
|
||||
match: /(?:^|\/)gpt-5\.6/,
|
||||
levels: LOW_TO_MAX,
|
||||
canDisable: true,
|
||||
offToken: 'none',
|
||||
completionsToolsNeedOff: true
|
||||
},
|
||||
{
|
||||
match: /(?:^|\/)gpt-5\.5/,
|
||||
levels: LOW_TO_XHIGH,
|
||||
canDisable: true,
|
||||
offToken: 'none',
|
||||
completionsToolsNeedOff: true
|
||||
},
|
||||
// A later gpt-5 minor keeps the tools limit, which holds for every version from 5.5,
|
||||
// but only the levels every gpt-5.x takes until it has a row of its own.
|
||||
{
|
||||
match: /(?:^|\/)gpt-5\.(?:[5-9]|\d{2,})/,
|
||||
levels: LOW_TO_HIGH,
|
||||
canDisable: true,
|
||||
offToken: 'none',
|
||||
completionsToolsNeedOff: true
|
||||
},
|
||||
// gpt-5.1+ are off only through `none`: omitted, they reason at medium.
|
||||
{ match: /(?:^|\/)gpt-5\./, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' },
|
||||
// gpt-5 and the o-series reject `none` and reason when it is omitted.
|
||||
{ match: /(?:^|\/)gpt-5(?:[:-]|$)/, levels: MINIMAL_TO_HIGH, canDisable: false },
|
||||
{ match: /(?:^|\/)o\d/, levels: LOW_TO_HIGH, canDisable: false }
|
||||
]
|
||||
|
||||
// Gemini 2.5/3 think by default; the backend proxy maps `none` to off on Flash, or to
|
||||
// the floor on Pro, which enforces one (level `low` on 3.x, 128 tokens on 2.5).
|
||||
// Gemini 3+ Flash / Flash-Lite accept `minimal`; 2.5 takes numeric budgets the proxy
|
||||
// maps from three tiers.
|
||||
const GEMINI_RULES: ReasoningRule[] = [
|
||||
{ match: /gemini-2\.5.*pro/, levels: LOW_TO_HIGH, canDisable: false, offToken: 'none' },
|
||||
{ match: /gemini-2\.5/, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' },
|
||||
{ match: /gemini-3.*pro/, levels: LOW_TO_HIGH, canDisable: false, offToken: 'none' },
|
||||
{ match: /gemini-3.*(flash|lite)/, levels: MINIMAL_TO_HIGH, canDisable: true, offToken: 'none' },
|
||||
{ match: /gemini-3/, levels: LOW_TO_HIGH, canDisable: true, offToken: 'none' }
|
||||
]
|
||||
|
||||
const REASONING_RULES: Partial<Record<AIProvider, ReasoningRule[]>> = {
|
||||
anthropic: ANTHROPIC_RULES,
|
||||
aws_bedrock: BEDROCK_RULES,
|
||||
openai: OPENAI_RULES,
|
||||
azure_openai: OPENAI_RULES,
|
||||
googleai: GEMINI_RULES,
|
||||
deepseek: [
|
||||
// Every current API model takes reasoning_effort, but only two levels are real:
|
||||
// low/medium are server-mapped to high, xhigh to max. The retired
|
||||
// `deepseek-chat` alias means "non-thinking mode", so a saved selection on it
|
||||
// must not silently become a thinking request. Off is a separate `thinking` param.
|
||||
{
|
||||
match: /(?:^|\/)deepseek(?!-chat(?::|$))/,
|
||||
levels: ['high', 'max'],
|
||||
canDisable: true,
|
||||
offToken: DEEPSEEK_OFF_SENTINEL
|
||||
}
|
||||
],
|
||||
mistral: [
|
||||
// Only the ids verified to accept reasoning_effort (large, magistral, ministral
|
||||
// and pinned versions reject it), whose only effort besides off is `high`.
|
||||
{
|
||||
match: /(?:^|\/)mistral-(?:(?:small|medium)-latest(?::|$)|medium-3[-.]5)/,
|
||||
levels: ['high'],
|
||||
canDisable: true
|
||||
}
|
||||
],
|
||||
// OpenRouter validates effort against its own vocabulary (minimal..xhigh + none) and
|
||||
// translates it per underlying provider, which scopes the ladder: Anthropic gets all
|
||||
// five as budget ratios, OpenAI gets the token verbatim, DeepSeek server-maps. `none`
|
||||
// is its documented off, more reliable than omission; it can only disable a model
|
||||
// whose upstream can.
|
||||
openrouter: [
|
||||
{
|
||||
match: /claude-(opus|sonnet)-5[-.][1-9](?!\d)/,
|
||||
levels: ['minimal', ...LOW_TO_XHIGH],
|
||||
canDisable: false
|
||||
},
|
||||
{
|
||||
match: /claude-(opus|sonnet)-(4|5)/,
|
||||
levels: ['minimal', ...LOW_TO_XHIGH],
|
||||
canDisable: true,
|
||||
offToken: 'none'
|
||||
},
|
||||
...GEMINI_RULES,
|
||||
...OPENAI_RULES.map(({ completionsToolsNeedOff: _, ...rule }) => ({
|
||||
...rule,
|
||||
offToken: 'none'
|
||||
})),
|
||||
{ match: /deepseek-v4/, levels: LOW_TO_XHIGH.slice(2), canDisable: true, offToken: 'none' },
|
||||
// deepseek-r1, grok-4 and :thinking variants reason unconditionally.
|
||||
{
|
||||
match: /deepseek-r|grok-4|:thinking/,
|
||||
levels: LOW_TO_HIGH,
|
||||
canDisable: false,
|
||||
offToken: 'none'
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
/**
|
||||
* Gemini Pro models cannot turn thinking off — the API enforces a floor
|
||||
* (level `low` on 3.x Pro, a 128-token budget on 2.5 Pro), so an off option
|
||||
* would silently mean "lowest". Flash / Flash-Lite can truly disable
|
||||
* (budget 0 / level `minimal`).
|
||||
*/
|
||||
function geminiCanDisable(model: string): boolean {
|
||||
return !model.toLowerCase().includes('pro')
|
||||
/** The row that describes a model, and whether its provider family has rows at all. */
|
||||
function findReasoningRule(
|
||||
provider: AIProvider,
|
||||
model: string
|
||||
): { rule: ReasoningRule | undefined; known: boolean } {
|
||||
const rules = REASONING_RULES[reasoningProviderFamily(provider, model)]
|
||||
const id = stripLegacyThinkingSuffix(model).toLowerCase()
|
||||
return { rule: rules?.find((rule) => rule.match.test(id)), known: rules !== undefined }
|
||||
}
|
||||
|
||||
/**
|
||||
* Anthropic's effort ladder is model-dependent: `xhigh` exists on Opus 4.7/4.8,
|
||||
* the 5 family and Fable/Mythos; `max` also on Opus 4.6 and Sonnet 4.6.
|
||||
* Offering an unsupported level would 400, so scope the list to the model.
|
||||
*/
|
||||
function anthropicReasoningLevels(model: string): ReasoningEffort[] {
|
||||
const m = model.toLowerCase()
|
||||
if (
|
||||
/claude-(opus|sonnet)-5/.test(m) ||
|
||||
/claude-opus-4-(7|8)/.test(m) ||
|
||||
m.includes('fable') ||
|
||||
m.includes('mythos')
|
||||
) {
|
||||
return ['low', 'medium', 'high', 'xhigh', 'max']
|
||||
}
|
||||
return ['low', 'medium', 'high', 'max']
|
||||
}
|
||||
|
||||
/** Mistral writes both `mistral-medium-3.5` and `mistral-medium-3-5`. */
|
||||
function normalizeMistralId(model: string): string {
|
||||
return baseModelId(model).replace(/\./g, '-')
|
||||
}
|
||||
|
||||
/**
|
||||
* Conservative static predicate for whether a model accepts an effort knob.
|
||||
* Kept tight to avoid 400s on models that reject reasoning params.
|
||||
*/
|
||||
function supportsReasoningStatic(provider: AIProvider, model: string): boolean {
|
||||
const m = model.toLowerCase()
|
||||
const base = baseModelId(model)
|
||||
switch (reasoningProviderFamily(provider, model)) {
|
||||
case 'anthropic':
|
||||
// Bedrock serves the same Claude models under prefixed ids
|
||||
// (e.g. `us.anthropic.claude-opus-4-6-v1`), so match on the full string.
|
||||
case 'aws_bedrock':
|
||||
// 4.6+ only: Opus 4.5 rejects adaptive thinking (and, on Bedrock,
|
||||
// the whole output_config surface) — live-verified hard 400.
|
||||
return (
|
||||
/claude-opus-(4-(6|7|8)|5)/.test(m) ||
|
||||
/claude-sonnet-(4-6|5)/.test(m) ||
|
||||
m.includes('fable') ||
|
||||
m.includes('mythos')
|
||||
)
|
||||
case 'openai':
|
||||
case 'azure_openai':
|
||||
return base.startsWith('gpt-5') || /^o\d/.test(base)
|
||||
case 'openrouter':
|
||||
// Best-effort markers for models whose `supported_parameters` include
|
||||
// `reasoning` in OpenRouter's catalog; OpenRouter translates the effort
|
||||
// per underlying provider.
|
||||
return (
|
||||
base.startsWith('gpt-5') ||
|
||||
/^o\d/.test(base) ||
|
||||
/claude-(opus|sonnet)-(4|5)/.test(m) ||
|
||||
/gemini-(2\.5|3)/.test(m) ||
|
||||
m.includes('deepseek-r') ||
|
||||
m.includes('deepseek-v4') ||
|
||||
m.includes('grok-4') ||
|
||||
m.includes(':thinking')
|
||||
)
|
||||
case 'googleai':
|
||||
return /gemini-(2\.5|3)/.test(m)
|
||||
case 'deepseek':
|
||||
// All current API models take reasoning_effort (live-verified). The
|
||||
// retired `deepseek-chat` alias stays excluded: its documented meaning
|
||||
// is "non-thinking mode", so a saved selection on it must not silently
|
||||
// become a thinking request.
|
||||
return base.startsWith('deepseek') && base !== 'deepseek-chat'
|
||||
case 'mistral':
|
||||
// Only the ids verified to accept reasoning_effort; other models
|
||||
// (large, magistral, ministral, pinned versions) reject the param.
|
||||
return (
|
||||
/^mistral-(small|medium)-latest$/.test(base) ||
|
||||
normalizeMistralId(model).startsWith('mistral-medium-3-5')
|
||||
)
|
||||
default:
|
||||
return false
|
||||
}
|
||||
/** Whether Chat Completions needs reasoning off for this model to take function tools. */
|
||||
export function completionsRejectsToolsWithReasoning(provider: AIProvider, model: string): boolean {
|
||||
return findReasoningRule(provider, model).rule?.completionsToolsNeedOff ?? false
|
||||
}
|
||||
|
||||
export type ReasoningCapability = {
|
||||
@@ -241,89 +282,12 @@ export type ReasoningCapability = {
|
||||
known: boolean
|
||||
}
|
||||
|
||||
/** Provider families the registry has real rules for; everything else is a shrug. */
|
||||
const KNOWN_REASONING_FAMILIES: ReadonlySet<string> = new Set([
|
||||
'anthropic',
|
||||
'aws_bedrock',
|
||||
'openai',
|
||||
'azure_openai',
|
||||
'openrouter',
|
||||
'googleai',
|
||||
'deepseek',
|
||||
'mistral'
|
||||
])
|
||||
|
||||
/** Resolve the reasoning capability of a model from the static registry. */
|
||||
export function getReasoningCapability(provider: AIProvider, model: string): ReasoningCapability {
|
||||
const bareModel = stripLegacyThinkingSuffix(model)
|
||||
const known = KNOWN_REASONING_FAMILIES.has(reasoningProviderFamily(provider, bareModel))
|
||||
const supported = supportsReasoningStatic(provider, bareModel)
|
||||
if (!supported) {
|
||||
return { supported: false, levels: [], canDisable: false, known }
|
||||
}
|
||||
const family = reasoningProviderFamily(provider, bareModel)
|
||||
const levels =
|
||||
family === 'anthropic' || family === 'aws_bedrock'
|
||||
? anthropicReasoningLevels(bareModel)
|
||||
: family === 'googleai'
|
||||
? geminiReasoningLevels(bareModel)
|
||||
: family === 'openai' || family === 'azure_openai'
|
||||
? openaiReasoningLevels(bareModel)
|
||||
: family === 'openrouter'
|
||||
? openrouterReasoningLevels(bareModel)
|
||||
: (PROVIDER_REASONING_LEVELS[family] ?? ['low', 'medium', 'high'])
|
||||
return { supported, levels, canDisable: canDisableReasoning(provider, bareModel), known }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether selecting "off" truly disables reasoning for the model. Off is
|
||||
* sent either as an explicit provider disable (see `explicitOffToken`) or by
|
||||
* omitting the effort — which only works where the model doesn't reason by
|
||||
* default.
|
||||
*/
|
||||
function canDisableReasoning(provider: AIProvider, model: string): boolean {
|
||||
const m = model.toLowerCase()
|
||||
const base = baseModelId(model)
|
||||
switch (reasoningProviderFamily(provider, model)) {
|
||||
case 'anthropic':
|
||||
// Every Claude but Fable and Mythos can stop thinking: 4.6-4.8 by
|
||||
// omission, and the 5 family through the explicit disable that
|
||||
// `explicitOffToken` sends.
|
||||
return !ANTHROPIC_ALWAYS_THINKING.test(m)
|
||||
case 'aws_bedrock':
|
||||
// Same models, different answer: AWS documents Sonnet 5 on Bedrock as
|
||||
// always thinking, where the native API accepts a disable for it.
|
||||
return !(ANTHROPIC_ALWAYS_THINKING.test(m) || m.includes('claude-sonnet-5'))
|
||||
case 'googleai':
|
||||
return geminiCanDisable(model)
|
||||
case 'openai':
|
||||
case 'azure_openai':
|
||||
// gpt-5.1+ accept effort 'none'; gpt-5 and o-series reject it and
|
||||
// reason at `medium` by default, so omission isn't off either.
|
||||
return /^gpt-5\./.test(base)
|
||||
case 'openrouter':
|
||||
// 'none' is in OpenRouter's vocabulary, but the gateway can't
|
||||
// disable a model whose upstream can't — scope off per underlying
|
||||
// family, like the levels.
|
||||
// The 5 family thinks by default, but its upstream takes an explicit
|
||||
// disable, so the gateway's 'none' has something to translate to.
|
||||
if (/claude-(opus|sonnet)-(4|5)/.test(m)) {
|
||||
return true
|
||||
}
|
||||
if (m.includes('gemini-')) {
|
||||
return geminiCanDisable(m)
|
||||
}
|
||||
if (base.startsWith('gpt-5') || /^o\d/.test(base)) {
|
||||
return /^gpt-5\./.test(base)
|
||||
}
|
||||
if (m.includes('deepseek-v4')) {
|
||||
return true
|
||||
}
|
||||
// grok-4, deepseek-r1 and :thinking variants reason unconditionally.
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
const { rule, known } = findReasoningRule(provider, model)
|
||||
return rule
|
||||
? { supported: true, levels: [...rule.levels], canDisable: rule.canDisable, known }
|
||||
: { supported: false, levels: [], canDisable: false, known }
|
||||
}
|
||||
|
||||
export function supportsReasoning(provider: AIProvider, model: string): boolean {
|
||||
@@ -352,64 +316,14 @@ export function resolveEffectiveReasoning(
|
||||
: undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Sentinel sent for the deepseek off case. It never reaches the wire as an
|
||||
* effort: the 'deepseek' branch of `applyReasoningToConfig` translates it to
|
||||
* the provider's `thinking: {type: "disabled"}` param (`reasoning_effort:
|
||||
* "none"` is rejected by their API).
|
||||
*/
|
||||
export const DEEPSEEK_OFF_SENTINEL: ReasoningEffort = 'none'
|
||||
|
||||
/**
|
||||
* Sentinel for the Anthropic off case. Like the DeepSeek one it never reaches
|
||||
* the wire as an effort: the 'anthropic' branch of `applyReasoningToConfig`
|
||||
* translates it to `thinking: {type: "disabled"}`, which is the only off the
|
||||
* always-on 5 family respects.
|
||||
*/
|
||||
export const ANTHROPIC_OFF_SENTINEL: ReasoningEffort = 'none'
|
||||
|
||||
/** Claude models whose thinking cannot be turned off — an explicit disable 400s. */
|
||||
const ANTHROPIC_ALWAYS_THINKING = /fable|mythos/
|
||||
|
||||
/**
|
||||
* Disable token to forward when the user explicitly turns reasoning off on a
|
||||
* model that reasons *by default* — omitting the field would silently keep
|
||||
* the default-on behavior. Undefined means omission is the correct off. Every
|
||||
* token is `'none'`: `requestsReasoning` reads that value as off.
|
||||
* the default-on behavior. Undefined means omission is the correct off, or that
|
||||
* the model cannot be turned off at all.
|
||||
*/
|
||||
export function explicitOffToken(provider: AIProvider, model: string): ReasoningEffort | undefined {
|
||||
switch (reasoningProviderFamily(provider, model)) {
|
||||
case 'anthropic':
|
||||
// Claude 4.6-4.8 only think when asked, so omission is already a
|
||||
// real off there and stays the wire form. Only the 5 family, which
|
||||
// thinks when the field is absent, needs the explicit disable —
|
||||
// Fable and Mythos reject it outright and get no off token at all.
|
||||
return /claude-(opus|sonnet)-5/.test(model.toLowerCase()) ? ANTHROPIC_OFF_SENTINEL : undefined
|
||||
case 'aws_bedrock':
|
||||
// Bedrock's Sonnet 5 cannot be disabled at all, so only Opus 5 gets
|
||||
// the sentinel; the rest keep omission.
|
||||
return model.toLowerCase().includes('claude-opus-5') ? ANTHROPIC_OFF_SENTINEL : undefined
|
||||
case 'googleai':
|
||||
// Gemini 2.5/3 think by default (dynamic budget / level). The backend
|
||||
// proxy maps 'none' to off on Flash, or the floor on Pro (only
|
||||
// reachable via a stale persisted preference — see canDisableReasoning).
|
||||
return 'none'
|
||||
case 'deepseek':
|
||||
return DEEPSEEK_OFF_SENTINEL
|
||||
case 'openai':
|
||||
case 'azure_openai':
|
||||
// gpt-5.1+ reasoning is off only via the explicit 'none' effort
|
||||
// (gpt-5.5 defaults to medium when the field is omitted).
|
||||
return /^gpt-5\./.test(baseModelId(model)) ? 'none' : undefined
|
||||
case 'openrouter':
|
||||
// OpenRouter validates effort against xhigh..minimal|none and
|
||||
// documents 'none' as disabling reasoning, translated per the
|
||||
// underlying provider — more reliable than omission, which keeps
|
||||
// reasoning-by-default models thinking.
|
||||
return 'none'
|
||||
default:
|
||||
return undefined
|
||||
}
|
||||
return findReasoningRule(provider, model).rule?.offToken
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"gitSyncTest": "hub/28950/git-repo-test-read-write-windmill",
|
||||
"gitInitRepo": "hub/28970/git-sync-init-repository-windmill",
|
||||
"gitInitRepo": "hub/28993/git-sync-init-repository-windmill",
|
||||
"slackErrorHandler": "hub/28794/workspace-or-schedule-error-handler-slack",
|
||||
"emailErrorHandler": "hub/19795/workspace-or-error-handler-email",
|
||||
"slackRecoveryHandler": "hub/28791/slack/schedule-recovery-handler-slack",
|
||||
|
||||
Reference in New Issue
Block a user