This commit is contained in:
Ruben Fiszel
2026-05-15 14:16:30 +00:00
211 changed files with 13395 additions and 3159 deletions
+18 -6
View File
@@ -20,6 +20,8 @@ on:
type: string
default: ''
secrets:
OPENAI_API_KEY:
required: false
CODEX_AUTH_JSON:
required: false
WINDMILL_EE_PRIVATE_ACCESS:
@@ -60,13 +62,18 @@ jobs:
- name: Check Codex configuration
id: codex_config
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
run: |
if [ -n "$CODEX_AUTH_JSON" ]; then
if [ -n "$OPENAI_API_KEY" ]; then
echo "enabled=true" >> "$GITHUB_OUTPUT"
echo "auth_mode=api_key" >> "$GITHUB_OUTPUT"
elif [ -n "$CODEX_AUTH_JSON" ]; then
echo "enabled=true" >> "$GITHUB_OUTPUT"
echo "auth_mode=oauth_json" >> "$GITHUB_OUTPUT"
else
echo "enabled=false" >> "$GITHUB_OUTPUT"
echo "CODEX_AUTH_JSON is not configured; skipping Codex review."
echo "Codex auth is not configured; set OPENAI_API_KEY or CODEX_AUTH_JSON to enable Codex review."
fi
- name: Resolve PR metadata
@@ -169,9 +176,10 @@ jobs:
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
run: npm install --global @openai/codex@0.128.0
- name: Configure file-backed Codex auth
- name: Configure Codex auth
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
run: |
CODEX_HOME="$HOME/.codex"
@@ -181,9 +189,13 @@ jobs:
cat > "$CODEX_HOME/config.toml" <<'EOF'
cli_auth_credentials_store = "file"
EOF
printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json"
chmod 600 "$CODEX_HOME/auth.json"
node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json"
if [ -n "$OPENAI_API_KEY" ]; then
printf '%s' "$OPENAI_API_KEY" | codex login --with-api-key
else
printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json"
chmod 600 "$CODEX_HOME/auth.json"
node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json"
fi
- name: Pre-fetch base and head refs for the PR
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
+1
View File
@@ -100,6 +100,7 @@ jobs:
extra_prompt: ${{ needs.parse.outputs.extra_prompt }}
triggered_by: ${{ github.event.comment.user.login }}
secrets:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
+84
View File
@@ -0,0 +1,84 @@
name: Publish CLI docs repo
# Regenerates the windmill-cli-docs repo (consumed by context7) from the
# canonical sources in this repo on every Windmill release.
#
# Required secret:
# CLI_DOCS_DEPLOY_KEY — ed25519 private key whose public half is registered
# as a write-access deploy key on
# windmill-labs/windmill-cli-docs.
on:
push:
tags:
- "v*"
workflow_dispatch:
# Serialize pushes to windmill-cli-docs so two release tags landing close
# together (e.g. a release-please bump + a hotfix) can't race to force-push
# the docs repo.
concurrency:
group: publish-cli-docs
cancel-in-progress: false
jobs:
publish:
runs-on: ubuntu-latest
steps:
- name: Checkout windmill (source of truth)
uses: actions/checkout@v4
with:
path: windmill
- name: Checkout windmill-cli-docs (publish target)
uses: actions/checkout@v4
with:
repository: windmill-labs/windmill-cli-docs
path: windmill-cli-docs
ssh-key: ${{ secrets.CLI_DOCS_DEPLOY_KEY }}
fetch-depth: 0
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
run: pip install pyyaml
- name: Regenerate docs
run: |
python3 windmill/system_prompts/generate.py \
--context7-dir "$GITHUB_WORKSPACE/windmill-cli-docs"
- name: Commit and push if changed
working-directory: windmill-cli-docs
env:
REF_NAME: ${{ github.ref_name }}
REF_TYPE: ${{ github.ref_type }}
run: |
git config user.name "windmill-bot"
git config user.email "bot@windmill.dev"
git add -A
if git diff --cached --quiet; then
echo "No doc changes for ${REF_NAME}."
committed=false
else
committed=true
if [ "${REF_TYPE}" = "tag" ]; then
git commit -m "chore: sync from windmill ${REF_NAME}"
else
git commit -m "chore: sync from windmill (manual dispatch from ${REF_NAME})"
fi
git push origin HEAD
fi
# Always mirror the version tag on tag pushes, even when content
# didn't change — downstream consumers tie snapshots to releases by
# tag, and skipping it would leave the docs repo without a tag for
# the new Windmill release.
# workflow_dispatch from a non-tag ref skips this so we don't
# create a junk tag named after a branch.
if [ "${REF_TYPE}" = "tag" ]; then
git tag -f "${REF_NAME}"
git push origin "${REF_NAME}" --force
echo "Mirrored tag ${REF_NAME} to windmill-cli-docs (content changed: ${committed})."
fi
+1
View File
@@ -20,6 +20,7 @@ rust-client/Cargo.toml
# Worktree-specific Claude Code settings (generated by scripts/worktree-env)
.claude/settings.local.json
.claude/worktrees/
# Symlinked cache directories (for git worktrees)
backend/target
+83
View File
@@ -1,5 +1,88 @@
# Changelog
## [1.702.1](https://github.com/windmill-labs/windmill/compare/v1.702.0...v1.702.1) (2026-05-14)
### Bug Fixes
* **nativets:** pass tracing-enabled OtelConfig to deno_telemetry::init ([#9163](https://github.com/windmill-labs/windmill/issues/9163)) ([bf99283](https://github.com/windmill-labs/windmill/commit/bf99283c3333bcdbc7679f4aea04ba29e41a48a5))
## [1.702.0](https://github.com/windmill-labs/windmill/compare/v1.701.0...v1.702.0) (2026-05-14)
### Features
* **git-sync:** sync extra_perms for flows/scripts/apps ([#9162](https://github.com/windmill-labs/windmill/issues/9162)) ([5e909b2](https://github.com/windmill-labs/windmill/commit/5e909b2b4f2819f19deaf06d9e78e6458b324683))
* include service accounts in instance settings users list ([#9157](https://github.com/windmill-labs/windmill/issues/9157)) ([e5286f4](https://github.com/windmill-labs/windmill/commit/e5286f46074cf2893e6ccd26175f929f16011c8f))
### Bug Fixes
* **mcp:** sanitize and enrich nested resource schemas ([#9158](https://github.com/windmill-labs/windmill/issues/9158)) ([d870edc](https://github.com/windmill-labs/windmill/commit/d870edc959481a06c894b4eda5e2be1a0269d7d0))
## [1.701.0](https://github.com/windmill-labs/windmill/compare/v1.700.2...v1.701.0) (2026-05-13)
### Features
* **frontend:** unified EditorHeader with file picker for flow/script/app editors ([#9047](https://github.com/windmill-labs/windmill/issues/9047)) ([d0f23cc](https://github.com/windmill-labs/windmill/commit/d0f23cc5238b025208c61e983701894de28536d5))
* read-only flag on API tokens ([#9144](https://github.com/windmill-labs/windmill/issues/9144)) ([d666e84](https://github.com/windmill-labs/windmill/commit/d666e8431cdbf14d9373d9ef625b5aafc50ac50a))
### Bug Fixes
* align script path existence check with deploy logic; hide Delete for non-admin ([#9152](https://github.com/windmill-labs/windmill/issues/9152)) ([c509206](https://github.com/windmill-labs/windmill/commit/c5092069cbeda2c4c18bea80dd629c7c087b30bf))
* Allow devops role to use all_workspaces runs filter in admins workspace ([#9153](https://github.com/windmill-labs/windmill/issues/9153)) ([110bef0](https://github.com/windmill-labs/windmill/commit/110bef0a6e76615c7b371c5c0f5bc1f4e7a73a64))
* **bun:** pass --preserve-symlinks on unbundled execution ([#9147](https://github.com/windmill-labs/windmill/issues/9147)) ([4d0f2c2](https://github.com/windmill-labs/windmill/commit/4d0f2c26a116a0f8a89a64231dc824eabda0a8c3))
* **cli:** prevent !inline-corruption in flow push/pull ([#9142](https://github.com/windmill-labs/windmill/issues/9142)) ([79c5b7b](https://github.com/windmill-labs/windmill/commit/79c5b7b8b7676b0a06fa6480dd04b7105d39d250))
* **operator:** refresh IAM RDS / Entra ID tokens in operator process ([#9141](https://github.com/windmill-labs/windmill/issues/9141)) ([7ebb081](https://github.com/windmill-labs/windmill/commit/7ebb08133cd4027bc00bacc4a0fc5865cd5709ec))
* **python:** preserve strings containing Infinity/NaN in result JSON ([#9149](https://github.com/windmill-labs/windmill/issues/9149)) ([33bf01b](https://github.com/windmill-labs/windmill/commit/33bf01b627c8ea430c03dfc27a97a8f2d770582f))
* scope promotion-mode debounce key per repo ([#9145](https://github.com/windmill-labs/windmill/issues/9145)) ([2ec1863](https://github.com/windmill-labs/windmill/commit/2ec1863340e759bba3408dbc4f41b16912b959ea))
* send flow push-loop ping outside transaction so zombie monitor sees it ([#9136](https://github.com/windmill-labs/windmill/issues/9136)) ([818cb31](https://github.com/windmill-labs/windmill/commit/818cb31fbc731fa5c70ddf5942bb37bc4bc56e4d))
### Performance Improvements
* **dynselect:** only retrigger when helper args actually change ([#9148](https://github.com/windmill-labs/windmill/issues/9148)) ([dd19e52](https://github.com/windmill-labs/windmill/commit/dd19e52a84fb9a9f48e3ad061b084841c2ee7464))
## [1.700.2](https://github.com/windmill-labs/windmill/compare/v1.700.1...v1.700.2) (2026-05-12)
### Bug Fixes
* preserve explicit nulls for typed fields in bulk instance config ([#9123](https://github.com/windmill-labs/windmill/issues/9123)) ([cab0000](https://github.com/windmill-labs/windmill/commit/cab0000f3a5e9a0b201a85da1a01b1f82df8a316))
* preserve negative integers in Bedrock tool schema conversion ([#9116](https://github.com/windmill-labs/windmill/issues/9116)) ([01e21c7](https://github.com/windmill-labs/windmill/commit/01e21c7f913eaf7dffc3d6a31501418ff2104c8b))
## [1.700.1](https://github.com/windmill-labs/windmill/compare/v1.700.0...v1.700.1) (2026-05-11)
### Bug Fixes
* CE build broken by enterprise-gated compute_instance_hash ([#9113](https://github.com/windmill-labs/windmill/issues/9113)) ([cd65de4](https://github.com/windmill-labs/windmill/commit/cd65de49285ff60abdd94c883180ded65609f382))
## [1.700.0](https://github.com/windmill-labs/windmill/compare/v1.699.0...v1.700.0) (2026-05-11)
### Features
* **cli:** auto-infer args for `wmill app push` ([#9091](https://github.com/windmill-labs/windmill/issues/9091)) ([43b1800](https://github.com/windmill-labs/windmill/commit/43b18006f32fd5db54bbf8ae7ff0e0b314a517e5))
* **forks:** prompt to delete forked children when deleting a fork ([#9097](https://github.com/windmill-labs/windmill/issues/9097)) ([e43a958](https://github.com/windmill-labs/windmill/commit/e43a958c5c6ae01a1fbecf3db63c6541a245be62))
* **operators:** allow operators to access assets page ([#9095](https://github.com/windmill-labs/windmill/issues/9095)) ([20ecd90](https://github.com/windmill-labs/windmill/commit/20ecd904e7060c3cf90f2605740bb349b2a3e6ed))
* **vault:** configurable JWT auth mount path and setup-doc fixes ([#9100](https://github.com/windmill-labs/windmill/issues/9100)) ([f8ba084](https://github.com/windmill-labs/windmill/commit/f8ba0840d74572c880cf458938365b3ec808c6fb))
### Bug Fixes
* add Input, Result, Trigger to reserved flow step IDs ([#9109](https://github.com/windmill-labs/windmill/issues/9109)) ([9f79a86](https://github.com/windmill-labs/windmill/commit/9f79a86a686708f66ccc512d4f132cb9a00397a7)), closes [#7139](https://github.com/windmill-labs/windmill/issues/7139)
* **frontend:** mark Path dirty when folder picker changes selection ([#9096](https://github.com/windmill-labs/windmill/issues/9096)) ([23bb1b5](https://github.com/windmill-labs/windmill/commit/23bb1b541e78846d5978153fd8d9bb4f01cec72b))
* mask oauth client secret in instance settings ([#9112](https://github.com/windmill-labs/windmill/issues/9112)) ([ac3c155](https://github.com/windmill-labs/windmill/commit/ac3c155541eb5ca20d65c38ad13dca6c10a572c9))
* populate raw_code for flowscript and appscript runs ([#9104](https://github.com/windmill-labs/windmill/issues/9104)) ([05172ac](https://github.com/windmill-labs/windmill/commit/05172ac3bdfc3472da5e9d8a825cdd479ba9e375))
### Performance Improvements
* lazy-load script editor history and hit partial index ([#9107](https://github.com/windmill-labs/windmill/issues/9107)) ([03e8bc8](https://github.com/windmill-labs/windmill/commit/03e8bc8c14258355d7d695333c1588807fbf8cd6))
## [1.699.0](https://github.com/windmill-labs/windmill/compare/v1.698.0...v1.699.0) (2026-05-08)
+11
View File
@@ -6,6 +6,7 @@ This folder contains black-box benchmark cases for:
- `app`
- `script`
- `cli`
- `global`
The goal is to test the current production prompts and guidance with realistic user requests, not to test one exact implementation shape.
@@ -75,6 +76,16 @@ Still, avoid benchmark phrasing. The prompt should read like a repo task, not a
When relevant, ask the assistant to tell the user which `wmill` commands to run next. That is part of the benchmarked behavior.
## Global-specific rules
Global prompts should exercise workspace-level drafting behavior:
- inspecting existing scripts, flows, apps, schedules, triggers, resources, and variables when relevant
- writing AI drafts rather than saving or deploying by default
- producing coherent multi-artifact changes when the request crosses artifact boundaries
Keep deterministic validation focused on the draft contract: required draft type/path, required content snippets, forbidden draft paths, and forbidden mutating tools such as deploy/delete unless the case explicitly asks for them.
## Deterministic validation
Use deterministic validation only for hard failures such as:
+21 -13
View File
@@ -1,11 +1,12 @@
# AI Evals
Small benchmark runner for the four Windmill AI generation modes:
Small benchmark runner for the Windmill AI generation modes:
- `cli`
- `flow`
- `script`
- `app`
- `global`
The benchmark always tests the current production prompts, tools, and guidance in this checkout.
@@ -55,8 +56,9 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus
bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o
bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose
bun run cli -- run flow --record
GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro --transport proxy
GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro
WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview
bun run cli -- run global global-test1-script-create
bun run cli -- run cli bun-hello-script
```
@@ -72,7 +74,6 @@ Public CLI surface:
- `--output <path>`: custom result JSON path
- `--model <alias>`: choose the model under test
- `--models <a,b,c>`: run the same cases sequentially against several model aliases
- `--transport <mode>`: frontend request transport (`direct` by default, `proxy` to exercise `/api/w/{workspace}/ai/proxy`)
- `--verbose`: stream assistant output for frontend runs
- `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl` for full-suite runs only
- `--backend-validation <mode>`: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals
@@ -95,7 +96,7 @@ Today:
Notes:
- the command also prints accepted alias spellings such as `gpt-4o`, `claude-opus-4.6`, and `claude-haiku-4.5`
- frontend modes (`flow`, `script`, `app`) can use Anthropic, OpenAI, and Gemini-backed aliases
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, and Gemini-backed aliases
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
- the judge model is separate and currently defaults to `claude-sonnet-4-6`
@@ -134,6 +135,13 @@ For `app` mode, `validate` can express narrow hard requirements such as:
- minimum datatable / datatable-table counts
- specific required datatable tables
For `global` mode, `validate` can express draft-level requirements such as:
- required draft type/path/language
- required or forbidden snippets in draft values
- required or forbidden draft counts
- forbidden draft paths
App fixtures can also include an optional `datatables.json` file at the fixture root.
For `flow` mode, an `initial` fixture can also include a benchmark workspace catalog of
@@ -145,26 +153,23 @@ If `--backend-validation preview` is enabled:
- `script` evals run a real backend script preview in an isolated temp workspace
- `flow` evals run a real backend flow preview only for cases that define `runtime.backendPreview`
- `flow` cases with `initial.workspace` fixtures seed those scripts and flows into the preview workspace before preview
- when `WMILL_AI_EVAL_BACKEND_WORKSPACE` is set, `ai_evals` treats that workspace as a dedicated test workspace, clears managed eval assets under `f/evals/*` before each preview run, and then reseeds the current case fixtures
- when `WMILL_AI_EVAL_BACKEND_WORKSPACE` is set, `ai_evals` creates or reuses that workspace as a dedicated test workspace, clears managed eval assets under `f/evals/*` before each preview run, and then reseeds the current case fixtures
Supported backend validation env vars:
Supported backend env vars:
- `WMILL_AI_EVAL_BACKEND_VALIDATION=preview`
- `WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000`
- `WMILL_AI_EVAL_BACKEND_EMAIL=admin@windmill.dev`
- `WMILL_AI_EVAL_BACKEND_PASSWORD=changeme`
- `WMILL_AI_EVAL_BACKEND_WORKSPACE=integration-tests` to reuse an existing workspace on CE installs with low workspace limits
- `WMILL_AI_EVAL_KEEP_WORKSPACES=1`
- `WMILL_AI_EVAL_WORKSPACE_PREFIX=ai-evals`
Frontend proxy transport uses the same backend auth/workspace env vars.
Frontend modes require a reachable Windmill backend and send model requests through the workspace AI proxy at `/api/w/{workspace}/ai/proxy`. At startup, `ai_evals` checks the resolved backend URL and fails early with setup guidance if the backend cannot be reached or login fails.
When `--transport proxy` is set:
For frontend modes:
- `ai_evals` creates or reuses a backend workspace
- `ai_evals` creates a temporary backend workspace, or creates/reuses `WMILL_AI_EVAL_BACKEND_WORKSPACE` when it is set
- it upserts a provider resource under `f/evals/ai/<provider>`
- frontend requests go through `/api/w/{workspace}/ai/proxy`
- result JSON and history records include `transport` so direct vs proxy runs stay distinguishable
## Results And Artifacts
@@ -178,11 +183,12 @@ If `--record` is used, the CLI also appends one compact JSON line to:
- `ai_evals/history/flow.jsonl`
- `ai_evals/history/script.jsonl`
- `ai_evals/history/app.jsonl`
- `ai_evals/history/global.jsonl`
- `ai_evals/history/cli.jsonl`
Each recorded line contains:
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `transport`, `judgeModel`)
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`)
- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`)
- average token usage (`averageTokenUsagePerAttempt`)
- per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate)
@@ -198,6 +204,7 @@ Typical artifacts by mode:
- `flow`: `flow.json`
- `script`: `script.json` plus the generated script file
- `app`: `app.json` plus frontend/backend files
- `global`: `global-drafts.json`
- `cli`: `assistant-output.txt`, `trace.json`, `wmill-invocations.jsonl`, plus generated workspace files
- backend-validated attempts also include `backend-preview.json`
@@ -213,6 +220,7 @@ Typical artifacts by mode:
## Notes
- Frontend modes reuse the production frontend chat code through the Vitest bridge.
- Global mode evaluates the production global AI tools and validates the resulting AI draft store.
- CLI mode creates an isolated workspace, writes the current checkout guidance into it, and benchmarks the real skills / `AGENTS.md` flow.
- CLI mode now also records a structured trace of invoked skills, tool calls, proposed `wmill` commands, and any attempted `wmill` executions.
- Frontend progress streams live while the benchmark is running.
@@ -210,8 +210,6 @@ function buildSettings(
baseUrl: 'http://backend.test/default',
email: 'admin@windmill.dev',
password: 'changeme',
keepWorkspaces: true,
workspacePrefix: 'ai-evals',
pollIntervalMs: 1,
maxWaitMs: 50,
...overrides
+5 -4
View File
@@ -24,6 +24,7 @@ export interface CompletedPreviewJob {
const tokenCache = new Map<string, Promise<string>>()
const sharedWorkspaceQueue = new Map<string, Promise<void>>()
const managedSharedWorkspacePrefixes = ['f/evals/']
const DEFAULT_WORKSPACE_PREFIX = 'ai-evals'
export class BackendPreviewClient {
constructor(private readonly settings: BackendValidationSettings) {}
@@ -35,7 +36,7 @@ export class BackendPreviewClient {
): Promise<T> {
const workspaceId =
this.settings.workspaceOverride ??
buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt)
buildWorkspaceId(caseId, attempt)
const run = async () => {
await this.ensureWorkspace(workspaceId)
@@ -46,7 +47,7 @@ export class BackendPreviewClient {
try {
return await body(workspaceId)
} finally {
if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) {
if (!this.settings.workspaceOverride) {
await this.deleteWorkspace(workspaceId).catch(() => undefined)
}
}
@@ -440,14 +441,14 @@ async function withSharedWorkspaceLock<T>(workspaceId: string, body: () => Promi
}
}
function buildWorkspaceId(prefix: string, caseId: string, attempt: number): string {
function buildWorkspaceId(caseId: string, attempt: number): string {
const caseSlug = caseId
.toLowerCase()
.replace(/[^a-z0-9-]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 30)
const suffix = randomUUID().slice(0, 8)
return `${prefix}-${caseSlug || 'case'}-a${attempt}-${suffix}`
return `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || 'case'}-a${attempt}-${suffix}`
}
function extractFolderName(path: string): string | null {
+12 -13
View File
@@ -1,6 +1,5 @@
import { loadSelectedCases } from "../../core/cases";
import { resolveBackendValidationSettings } from "../../core/backendValidation";
import { resolveFrontendEvalTransportSettings } from "../../core/frontendTransport";
import {
formatRunModelLabel,
getFrontendEvalModel,
@@ -9,13 +8,15 @@ import {
import { buildRunResult } from "../../core/results";
import { runSuite } from "../../core/runSuite";
import type { BenchmarkRunResult, ModeRunner } from "../../core/types";
import { resolveWindmillBackendSettings } from "../../core/windmillBackendSettings";
import { emitFrontendBenchmarkProgress } from "./progress";
import { createAppModeRunner } from "../../modes/app";
import { createFlowModeRunner } from "../../modes/flow";
import { createGlobalModeRunner } from "../../modes/global";
import { createScriptModeRunner } from "../../modes/script";
import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
export type FrontendBenchmarkMode = "flow" | "app" | "script";
export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global";
export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult> {
const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE);
@@ -36,17 +37,14 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
evalMode: mode,
requestedMode: process.env.WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION,
});
const transportSettings = resolveFrontendEvalTransportSettings({
evalMode: mode,
requestedTransport: process.env.WMILL_FRONTEND_AI_EVAL_TRANSPORT,
});
const backendSettings = resolveWindmillBackendSettings();
const selectedCases = await loadSelectedCases(mode, caseIds);
const modeRunner = getModeRunner(
mode,
getFrontendEvalModel(model),
backendValidation,
transportSettings,
backendSettings,
);
const runModel = formatRunModelLabel(mode, model);
const caseResults = await runSuite({
@@ -66,7 +64,6 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
mode,
runs,
runModel,
transport: transportSettings.transport,
judgeModel: DEFAULT_JUDGE_MODEL,
caseResults,
});
@@ -76,24 +73,26 @@ function getModeRunner(
mode: FrontendBenchmarkMode,
model: ReturnType<typeof getFrontendEvalModel>,
backendValidation: ReturnType<typeof resolveBackendValidationSettings>,
transportSettings: ReturnType<typeof resolveFrontendEvalTransportSettings>,
backendSettings: ReturnType<typeof resolveWindmillBackendSettings>,
): ModeRunner<any, any, any> {
switch (mode) {
case "flow":
return createFlowModeRunner(model, backendValidation, transportSettings);
return createFlowModeRunner(model, backendValidation, backendSettings);
case "app":
return createAppModeRunner(model, transportSettings);
return createAppModeRunner(model, backendSettings);
case "script":
return createScriptModeRunner(
model,
backendValidation,
transportSettings,
backendSettings,
);
case "global":
return createGlobalModeRunner(model, backendSettings);
}
}
function parseMode(value: string | undefined): FrontendBenchmarkMode {
if (value === "flow" || value === "app" || value === "script") {
if (value === "flow" || value === "app" || value === "script" || value === "global") {
return value;
}
throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`);
@@ -12,7 +12,7 @@ import {
prepareAppUserMessage,
} from "../../../../../frontend/src/lib/components/copilot/chat/app/core";
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
import { createAppFileHelpers } from "./fileHelpers";
import { createAppFileHelpers, type AppEvalChatHelpers } from "./fileHelpers";
import { runEval } from "../shared";
import type { AIProvider } from "$lib/gen/types.gen";
import type {
@@ -22,7 +22,6 @@ import type {
} from "../../../../core/types";
import type { TokenUsage } from "../shared/types";
import type { AppFilesState } from "../../../../core/validators";
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
import {
createAppBackendRunnableContextElement,
@@ -49,8 +48,7 @@ export interface AppEvalOptions {
model?: string;
maxIterations?: number;
provider?: AIProvider;
transport?: FrontendEvalTransport;
backend?: WindmillBackendSettings;
backend: WindmillBackendSettings;
workspaceRoot?: string;
runContext?: ModeRunContext;
}
@@ -58,7 +56,7 @@ export interface AppEvalOptions {
export async function runAppEval(
userPrompt: string,
apiKey: string,
options?: AppEvalOptions,
options: AppEvalOptions,
): Promise<AppEvalResult> {
const workspaceRoot =
options?.workspaceRoot ??
@@ -101,10 +99,9 @@ export async function runAppEval(
model,
workspace: workspaceRoot,
provider: options?.provider,
transport: options?.transport,
backend: options?.backend,
proxyCaseId: options?.runContext?.caseId,
proxyAttempt: options?.runContext?.attempt,
backend: options.backend,
caseId: options?.runContext?.caseId,
attempt: options?.runContext?.attempt,
},
});
@@ -124,7 +121,7 @@ export async function runAppEval(
async function buildAdditionalContext(
appContext: EvalCaseRuntimeAppContextSpec | undefined,
helpers: AppAIChatHelpers,
helpers: AppEvalChatHelpers,
): Promise<ContextElement[]> {
const entries = appContext?.additional ?? [];
if (entries.length === 0) {
@@ -2,6 +2,7 @@ import { mkdir, rm, writeFile } from 'fs/promises'
import { dirname, join } from 'path'
import type {
AppAIChatHelpers,
AppDatatableMetadata,
AppFiles,
BackendRunnable,
DataTableSchema,
@@ -10,6 +11,10 @@ import type {
} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
import { buildAppWmillTypes, collectAppDiagnostics } from '../../../../core/appDiagnostics'
export interface AppEvalChatHelpers extends AppAIChatHelpers {
getDatatables: () => Promise<DataTableSchema[]>
}
async function writeFrontendFile(
workspaceRoot: string | undefined,
path: string,
@@ -92,7 +97,7 @@ export async function createAppFileHelpers(
initialDatatables: DataTableSchema[] = [],
workspaceRoot?: string
): Promise<{
helpers: AppAIChatHelpers
helpers: AppEvalChatHelpers
getFiles: () => AppFiles
getEvalState: () => {
frontend: Record<string, string>
@@ -137,7 +142,7 @@ export async function createAppFileHelpers(
}
await persistDatatables(workspaceRoot, datatables)
const helpers: AppAIChatHelpers = {
const helpers: AppEvalChatHelpers = {
listFrontendFiles: () => [
...Object.keys(frontend).filter((path) => path !== '/wmill.d.ts'),
'/wmill.d.ts'
@@ -211,6 +216,34 @@ export async function createAppFileHelpers(
},
lint,
getDatatables: async () => structuredClone(datatables),
listDatatableTables: async () =>
datatables.map(
(datatable): AppDatatableMetadata => {
const schemas = Object.fromEntries(
Object.entries(datatable.schemas).map(([schemaName, tables]) => [
schemaName,
Object.keys(tables)
])
)
return {
datatable_name: datatable.datatable_name,
schemas,
tableCount: Object.values(schemas).reduce(
(sum, tableNames) => sum + tableNames.length,
0
),
error: datatable.error
}
}
),
getDatatableTableSchema: async (
datatableName: string,
schemaName: string,
tableName: string
) => {
const datatable = datatables.find((entry) => entry.datatable_name === datatableName)
return structuredClone(datatable?.schemas?.[schemaName]?.[tableName] ?? {})
},
getAvailableDatatableNames: () => datatables.map((datatable) => datatable.datatable_name),
execDatatableSql: async (
datatableName: string,
@@ -18,7 +18,6 @@ import {
import { runEval } from "../shared";
import type { ModeRunContext } from "../../../../core/types";
import type { TokenUsage, ToolCallDetail } from "../shared/types";
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
export interface FlowFixture {
@@ -48,8 +47,7 @@ export interface FlowEvalOptions {
model?: string;
maxIterations?: number;
provider?: AIProvider;
transport?: FrontendEvalTransport;
backend?: WindmillBackendSettings;
backend: WindmillBackendSettings;
workspaceRoot?: string;
runContext?: ModeRunContext;
}
@@ -57,7 +55,7 @@ export interface FlowEvalOptions {
export async function runFlowEval(
userPrompt: string,
apiKey: string,
options?: FlowEvalOptions,
options: FlowEvalOptions,
): Promise<FlowEvalResult> {
const workspaceRoot =
options?.workspaceRoot ??
@@ -100,10 +98,9 @@ export async function runFlowEval(
model,
workspace: workspaceRoot,
provider: options?.provider,
transport: options?.transport,
backend: options?.backend,
proxyCaseId: options?.runContext?.caseId,
proxyAttempt: options?.runContext?.attempt,
backend: options.backend,
caseId: options?.runContext?.caseId,
attempt: options?.runContext?.attempt,
},
});
@@ -0,0 +1,127 @@
import { mkdtemp, rm } from "fs/promises";
import { tmpdir } from "os";
import { join } from "path";
import type { AIProvider } from "$lib/gen/types.gen";
import {
globalTools,
prepareGlobalSystemMessage,
prepareGlobalUserMessage,
} from "../../../../../frontend/src/lib/components/copilot/chat/global/core";
import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte";
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
import type { ModeRunContext } from "../../../../core/types";
import type { GlobalDraftState } from "../../../../core/validators";
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
import {
registerBenchmarkWorkspaceRunnables,
unregisterBenchmarkWorkspaceRunnables,
type BenchmarkWorkspaceRunnables,
} from "../../mockBackend";
import { runEval } from "../shared";
import type { TokenUsage, ToolCallDetail } from "../shared/types";
const MUTATING_GLOBAL_TOOLS = new Set([
"deploy_workspace_item",
"delete_workspace_item",
]);
export interface GlobalEvalResult {
success: boolean;
state: GlobalDraftState;
error?: string;
assistantMessageCount: number;
toolCallCount: number;
toolsUsed: string[];
toolCallDetails: ToolCallDetail[];
tokenUsage: TokenUsage;
}
export interface GlobalEvalOptions {
workspaceFixtures?: BenchmarkWorkspaceRunnables;
model?: string;
maxIterations?: number;
provider?: AIProvider;
backend: WindmillBackendSettings;
workspaceRoot?: string;
runContext?: ModeRunContext;
}
export async function runGlobalEval(
userPrompt: string,
apiKey: string,
options: GlobalEvalOptions,
): Promise<GlobalEvalResult> {
const workspaceRoot =
options.workspaceRoot ??
(await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-")));
globalDraftStore.clearDrafts(workspaceRoot);
registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {});
try {
const model = options.model ?? "claude-haiku-4-5-20251001";
const rawResult = await runEval({
userPrompt,
systemMessage: prepareGlobalSystemMessage(),
userMessage: prepareGlobalUserMessage(userPrompt),
tools: getGlobalEvalTools(),
helpers: {},
apiKey,
getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }),
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
onAssistantToken: options.runContext?.onAssistantChunk,
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
onToolCall: options.runContext?.onToolCall,
options: {
maxIterations: options.maxIterations,
model,
workspace: workspaceRoot,
provider: options.provider,
backend: options.backend,
caseId: options.runContext?.caseId,
attempt: options.runContext?.attempt,
},
});
return {
state: rawResult.output,
success: rawResult.success,
error: rawResult.error,
assistantMessageCount: rawResult.iterations,
toolCallCount: rawResult.toolCallsCount,
toolsUsed: rawResult.toolsCalled,
toolCallDetails: rawResult.toolCallDetails,
tokenUsage: rawResult.tokenUsage,
};
} finally {
globalDraftStore.clearDrafts(workspaceRoot);
unregisterBenchmarkWorkspaceRunnables(workspaceRoot);
if (!options.workspaceRoot) {
await rm(workspaceRoot, { recursive: true, force: true });
}
}
}
function getGlobalEvalTools(): ProductionTool<{}>[] {
return (globalTools as ProductionTool<{}>[]).map((tool) => {
if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) {
return tool;
}
return {
...tool,
requiresConfirmation: false,
validateBeforeConfirmation: undefined,
fn: async () =>
JSON.stringify(
{
success: false,
error:
"This mutating workspace tool is disabled during ai_evals global mode.",
},
null,
2,
),
};
});
}
@@ -14,7 +14,6 @@ import { createScriptFileHelpers, type ScriptEvalState } from "./fileHelpers";
import { runEval } from "../shared";
import type { ModeRunContext } from "../../../../core/types";
import type { TokenUsage, ToolCallDetail } from "../shared/types";
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
export interface ScriptEvalResult {
@@ -33,8 +32,7 @@ export interface ScriptEvalOptions {
model?: string;
maxIterations?: number;
provider?: AIProvider;
transport?: FrontendEvalTransport;
backend?: WindmillBackendSettings;
backend: WindmillBackendSettings;
workspaceRoot?: string;
runContext?: ModeRunContext;
}
@@ -98,10 +96,9 @@ export async function runScriptEval(
model,
workspace: workspaceRoot,
provider: modelProvider.provider,
transport: options.transport,
backend: options.backend,
proxyCaseId: options.runContext?.caseId,
proxyAttempt: options.runContext?.attempt,
caseId: options.runContext?.caseId,
attempt: options.runContext?.attempt,
},
});
@@ -40,8 +40,8 @@ export interface RunEvalParams<THelpers, TOutput> {
apiKey: string;
/** Function to get the current output state */
getOutput: () => TOutput;
/** Optional configuration */
options?: EvalRunnerOptions;
/** Model and Windmill backend configuration */
options: EvalRunnerOptions;
onAssistantMessageStart?: () => void;
onAssistantToken?: (token: string) => void;
onAssistantMessageEnd?: () => void;
@@ -70,10 +70,10 @@ export async function runEval<THelpers, TOutput>(
} = params;
let shouldEmitMessageStart = true;
const model = options?.model ?? "gpt-4o";
const maxIterations = options?.maxIterations ?? 20;
const workspace = options?.workspace ?? "test-workspace";
const provider = toFrontendEvalProvider(options?.provider);
const model = options.model ?? "gpt-4o";
const maxIterations = options.maxIterations ?? 20;
const workspace = options.workspace ?? "test-workspace";
const provider = toFrontendEvalProvider(options.provider);
const modelProvider = resolveEvalModelProvider(model, provider);
@@ -203,45 +203,31 @@ export async function runEval<THelpers, TOutput>(
}
};
if (options?.transport === "proxy") {
const backendSettings = options.backend;
if (!backendSettings) {
throw new Error("Missing backend settings for proxy transport");
}
const backendClient = new WindmillBackendClient(backendSettings);
return await backendClient.withWorkspace(
options.proxyCaseId ?? "eval",
options.proxyAttempt ?? 1,
async (proxyWorkspaceId) => {
const resourcePath = buildProxyResourcePath(modelProvider.provider);
await backendClient.upsertResource({
workspaceId: proxyWorkspaceId,
path: resourcePath,
resourceType: modelProvider.provider,
value: { api_key: apiKey },
});
const token = await backendClient.getToken();
const clients = createEvalClients({
provider: modelProvider.provider,
apiKey,
transport: "proxy",
proxy: {
baseURL: `${backendSettings.baseUrl}/api/w/${encodeURIComponent(proxyWorkspaceId)}/ai/proxy`,
bearerToken: token,
resourcePath,
},
}) as unknown as ChatClients;
return await executeChatLoop(clients);
},
);
}
const clients = createEvalClients({
provider: modelProvider.provider,
apiKey,
}) as unknown as ChatClients;
return await executeChatLoop(clients);
const backendSettings = options.backend;
const backendClient = new WindmillBackendClient(backendSettings);
return await backendClient.withWorkspace(
options.caseId ?? "eval",
options.attempt ?? 1,
async (proxyWorkspaceId) => {
const resourcePath = buildProxyResourcePath(modelProvider.provider);
await backendClient.upsertResource({
workspaceId: proxyWorkspaceId,
path: resourcePath,
resourceType: modelProvider.provider,
value: { api_key: apiKey },
});
const token = await backendClient.getToken();
const clients = createEvalClients({
provider: modelProvider.provider,
proxy: {
baseURL: `${backendSettings.baseUrl}/api/w/${encodeURIComponent(proxyWorkspaceId)}/ai/proxy`,
bearerToken: token,
resourcePath,
},
}) as unknown as ChatClients;
return await executeChatLoop(clients);
},
);
}
function toFrontendEvalProvider(
@@ -2,35 +2,9 @@ import { describe, expect, it } from "bun:test";
import {
buildProxyHeaders,
buildProxyResourcePath,
buildOpenAICompatibleClientOptions,
resolveEvalModelProvider,
} from "./providerConfig";
describe("buildOpenAICompatibleClientOptions", () => {
it("adds Gemini's OpenAI-compatible base URL and client header", () => {
const options = buildOpenAICompatibleClientOptions(
"googleai",
"gemini-test-key",
);
expect(options).toMatchObject({
apiKey: "gemini-test-key",
baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
defaultHeaders: {
"x-goog-api-client": "windmill-ai-evals/1.0",
},
});
});
it("keeps the default OpenAI-compatible config for OpenAI", () => {
expect(
buildOpenAICompatibleClientOptions("openai", "openai-test-key"),
).toEqual({
apiKey: "openai-test-key",
});
});
});
describe("proxy helpers", () => {
it("builds provider-scoped proxy resource paths", () => {
expect(buildProxyResourcePath("googleai")).toBe("f/evals/ai/googleai");
@@ -1,7 +1,6 @@
import Anthropic from "@anthropic-ai/sdk";
import OpenAI from "openai";
import type { FrontendEvalModelConfig } from "../../../../core/models";
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
export type FrontendEvalProvider = FrontendEvalModelConfig["provider"];
@@ -15,15 +14,12 @@ export interface ResolvedEvalModelProvider {
model: string;
}
export interface EvalProxyClientConfig {
export interface WindmillAiProxyClientConfig {
baseURL: string;
bearerToken: string;
resourcePath: string;
}
const GEMINI_OPENAI_BASE_URL =
"https://generativelanguage.googleapis.com/v1beta/openai/";
const GEMINI_GOOG_API_CLIENT = "windmill-ai-evals/1.0";
const EVAL_PROXY_RESOURCE_PREFIX = "f/evals/ai";
export function buildProxyHeaders(
@@ -40,25 +36,8 @@ export function buildProxyResourcePath(provider: FrontendEvalProvider): string {
return `${EVAL_PROXY_RESOURCE_PREFIX}/${provider}`;
}
export function buildOpenAICompatibleClientOptions(
provider: Exclude<FrontendEvalProvider, "anthropic">,
apiKey: string,
): ConstructorParameters<typeof OpenAI>[0] {
if (provider === "googleai") {
return {
apiKey,
baseURL: GEMINI_OPENAI_BASE_URL,
defaultHeaders: {
"x-goog-api-client": GEMINI_GOOG_API_CLIENT,
},
};
}
return { apiKey };
}
function buildProxyOpenAIClientOptions(
proxy: EvalProxyClientConfig,
proxy: WindmillAiProxyClientConfig,
): ConstructorParameters<typeof OpenAI>[0] {
return {
apiKey: "unused",
@@ -69,52 +48,24 @@ function buildProxyOpenAIClientOptions(
export function createEvalClients(input: {
provider: FrontendEvalProvider;
apiKey: string;
transport?: FrontendEvalTransport;
proxy?: EvalProxyClientConfig;
proxy: WindmillAiProxyClientConfig;
}): EvalClients {
const transport = input.transport ?? "direct";
if (input.provider === "anthropic") {
if (transport === "proxy") {
if (!input.proxy) {
throw new Error(
"Missing proxy client configuration for proxy transport",
);
}
return {
openai: new OpenAI({ apiKey: "unused" }),
anthropic: new Anthropic({
apiKey: "unused",
baseURL: input.proxy.baseURL,
defaultHeaders: buildProxyHeaders(
input.proxy.bearerToken,
input.proxy.resourcePath,
),
}),
};
}
return {
openai: new OpenAI({ apiKey: "unused" }),
anthropic: new Anthropic({ apiKey: input.apiKey }),
};
}
if (transport === "proxy") {
if (!input.proxy) {
throw new Error("Missing proxy client configuration for proxy transport");
}
return {
openai: new OpenAI(buildProxyOpenAIClientOptions(input.proxy)),
anthropic: new Anthropic({ apiKey: "unused" }),
anthropic: new Anthropic({
apiKey: "unused",
baseURL: input.proxy.baseURL,
defaultHeaders: buildProxyHeaders(
input.proxy.bearerToken,
input.proxy.resourcePath,
),
}),
};
}
return {
openai: new OpenAI(
buildOpenAICompatibleClientOptions(input.provider, input.apiKey),
),
openai: new OpenAI(buildProxyOpenAIClientOptions(input.proxy)),
anthropic: new Anthropic({ apiKey: "unused" }),
};
}
@@ -1,6 +1,5 @@
import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
import type { AIProvider } from "$lib/gen/types.gen";
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
export interface TokenUsage {
@@ -15,14 +14,13 @@ export interface ToolCallDetail {
}
export interface EvalRunnerOptions {
backend: WindmillBackendSettings;
maxIterations?: number;
model?: string;
workspace?: string;
provider?: AIProvider;
transport?: FrontendEvalTransport;
backend?: WindmillBackendSettings;
proxyCaseId?: string;
proxyAttempt?: number;
caseId?: string;
attempt?: number;
}
export interface RawEvalResult<TOutput> {
+1 -1
View File
@@ -1,4 +1,4 @@
export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script'
export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global'
export type FrontendBenchmarkProgressEvent =
| {
+1 -6
View File
@@ -16,14 +16,13 @@ const FRONTEND_BENCHMARK_TEST =
const FRONTEND_BENCHMARK_CONFIG =
"../ai_evals/adapters/frontend/vitest.config.ts";
export type FrontendMode = "flow" | "app" | "script";
export type FrontendMode = "flow" | "app" | "script" | "global";
export async function runFrontendBenchmarkAdapter(input: {
mode: FrontendMode;
caseIds: string[];
runs: number;
model?: string;
transport?: string;
verbose?: boolean;
backendValidation?: string;
}): Promise<BenchmarkRunResult> {
@@ -44,10 +43,6 @@ export async function runFrontendBenchmarkAdapter(input: {
WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "",
};
if (input.transport) {
env.WMILL_FRONTEND_AI_EVAL_TRANSPORT = input.transport;
}
try {
await runVitestBenchmark(
path.join(FRONTEND_DIR, "node_modules", ".bin", "vitest"),
@@ -65,6 +65,10 @@ vi.mock('$lib/gen', async () => {
hasBenchmarkWorkspace(data.workspace)
? (listBenchmarkScripts(data.workspace) ?? [])
: actual.ScriptService.listScripts(data),
existsScriptByPath: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace)
? Boolean(getBenchmarkScriptByPath(data.workspace, data.path))
: actual.ScriptService.existsScriptByPath(data),
getScriptByPath: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const script = getBenchmarkScriptByPath(data.workspace, data.path)
@@ -91,6 +95,10 @@ vi.mock('$lib/gen', async () => {
hasBenchmarkWorkspace(data.workspace)
? (listBenchmarkFlows(data.workspace) ?? [])
: actual.FlowService.listFlows(data),
existsFlowByPath: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace)
? Boolean(getBenchmarkFlowByPath(data.workspace, data.path))
: actual.FlowService.existsFlowByPath(data),
getFlowByPath: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
@@ -142,6 +150,16 @@ vi.mock('$lib/gen', async () => {
}
}),
ScheduleService: wrapService(actual.ScheduleService, {
existsSchedule: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.ScheduleService.existsSchedule(data),
listSchedules: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ScheduleService.listSchedules(data),
getSchedule: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`Schedule "${data.path}" not found in benchmark workspace`)
}
return actual.ScheduleService.getSchedule(data)
},
previewSchedule: async (data: { requestBody?: Record<string, unknown> }) =>
previewBenchmarkSchedule(data),
createSchedule: async (data: { workspace: string; requestBody: Record<string, unknown> }) =>
@@ -149,11 +167,167 @@ vi.mock('$lib/gen', async () => {
? createBenchmarkSchedule(data)
: actual.ScheduleService.createSchedule(data)
}),
ResourceService: wrapService(actual.ResourceService, {
existsResource: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.ResourceService.existsResource(data),
listResource: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.listResource(data),
getResource: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`Resource "${data.path}" not found in benchmark workspace`)
}
return actual.ResourceService.getResource(data)
},
queryResourceTypes: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data)
}),
VariableService: wrapService(actual.VariableService, {
existsVariable: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.VariableService.existsVariable(data),
listVariable: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.VariableService.listVariable(data),
getVariable: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`Variable "${data.path}" not found in benchmark workspace`)
}
return actual.VariableService.getVariable(data)
}
}),
AppService: wrapService(actual.AppService, {
existsApp: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.AppService.existsApp(data),
listApps: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.AppService.listApps(data),
getAppByPath: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`App "${data.path}" not found in benchmark workspace`)
}
return actual.AppService.getAppByPath(data)
}
}),
HttpTriggerService: wrapService(actual.HttpTriggerService, {
existsHttpTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.HttpTriggerService.existsHttpTrigger(data),
listHttpTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.HttpTriggerService.listHttpTriggers(data),
getHttpTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`HTTP trigger "${data.path}" not found in benchmark workspace`)
}
return actual.HttpTriggerService.getHttpTrigger(data)
},
createHttpTrigger: async (data: { workspace: string; requestBody: Record<string, unknown> }) =>
hasBenchmarkWorkspace(data.workspace)
? createBenchmarkHttpTrigger(data)
: actual.HttpTriggerService.createHttpTrigger(data)
}),
WebsocketTriggerService: wrapService(actual.WebsocketTriggerService, {
existsWebsocketTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace)
? false
: actual.WebsocketTriggerService.existsWebsocketTrigger(data),
listWebsocketTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace)
? []
: actual.WebsocketTriggerService.listWebsocketTriggers(data),
getWebsocketTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`Websocket trigger "${data.path}" not found in benchmark workspace`)
}
return actual.WebsocketTriggerService.getWebsocketTrigger(data)
}
}),
KafkaTriggerService: wrapService(actual.KafkaTriggerService, {
existsKafkaTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace)
? false
: actual.KafkaTriggerService.existsKafkaTrigger(data),
listKafkaTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.KafkaTriggerService.listKafkaTriggers(data),
getKafkaTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`Kafka trigger "${data.path}" not found in benchmark workspace`)
}
return actual.KafkaTriggerService.getKafkaTrigger(data)
}
}),
NatsTriggerService: wrapService(actual.NatsTriggerService, {
existsNatsTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.NatsTriggerService.existsNatsTrigger(data),
listNatsTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.NatsTriggerService.listNatsTriggers(data),
getNatsTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`NATS trigger "${data.path}" not found in benchmark workspace`)
}
return actual.NatsTriggerService.getNatsTrigger(data)
}
}),
PostgresTriggerService: wrapService(actual.PostgresTriggerService, {
existsPostgresTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace)
? false
: actual.PostgresTriggerService.existsPostgresTrigger(data),
listPostgresTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace)
? []
: actual.PostgresTriggerService.listPostgresTriggers(data),
getPostgresTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`Postgres trigger "${data.path}" not found in benchmark workspace`)
}
return actual.PostgresTriggerService.getPostgresTrigger(data)
}
}),
MqttTriggerService: wrapService(actual.MqttTriggerService, {
existsMqttTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.MqttTriggerService.existsMqttTrigger(data),
listMqttTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.MqttTriggerService.listMqttTriggers(data),
getMqttTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`MQTT trigger "${data.path}" not found in benchmark workspace`)
}
return actual.MqttTriggerService.getMqttTrigger(data)
}
}),
SqsTriggerService: wrapService(actual.SqsTriggerService, {
existsSqsTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.SqsTriggerService.existsSqsTrigger(data),
listSqsTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.SqsTriggerService.listSqsTriggers(data),
getSqsTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`SQS trigger "${data.path}" not found in benchmark workspace`)
}
return actual.SqsTriggerService.getSqsTrigger(data)
}
}),
GcpTriggerService: wrapService(actual.GcpTriggerService, {
existsGcpTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace) ? false : actual.GcpTriggerService.existsGcpTrigger(data),
listGcpTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.GcpTriggerService.listGcpTriggers(data),
getGcpTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`GCP trigger "${data.path}" not found in benchmark workspace`)
}
return actual.GcpTriggerService.getGcpTrigger(data)
}
}),
AzureTriggerService: wrapService(actual.AzureTriggerService, {
existsAzureTrigger: async (data: { workspace: string; path: string }) =>
hasBenchmarkWorkspace(data.workspace)
? false
: actual.AzureTriggerService.existsAzureTrigger(data),
listAzureTriggers: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace) ? [] : actual.AzureTriggerService.listAzureTriggers(data),
getAzureTrigger: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
throw new Error(`Azure trigger "${data.path}" not found in benchmark workspace`)
}
return actual.AzureTriggerService.getAzureTrigger(data)
}
})
}
})
@@ -0,0 +1,104 @@
import { afterEach, describe, expect, it } from "bun:test";
import type { WindmillBackendSettings } from "../../core/windmillBackendSettings";
import {
WindmillBackendClient,
assertWindmillBackendReachable,
} from "./windmillBackend";
const ORIGINAL_FETCH = globalThis.fetch;
afterEach(() => {
globalThis.fetch = ORIGINAL_FETCH;
});
describe("assertWindmillBackendReachable", () => {
it("logs in to verify backend reachability", async () => {
const requests: Array<{ url: string; init?: RequestInit }> = [];
globalThis.fetch = mockFetch(requests, textResponse(200, "token"));
await expect(
assertWindmillBackendReachable(
buildSettings({ baseUrl: "http://backend.test/reachable" }),
),
).resolves.toBeUndefined();
expect(requests.map((entry) => entry.url)).toEqual([
"http://backend.test/reachable/api/auth/login",
]);
});
it("adds setup guidance when the backend cannot be initialized", async () => {
globalThis.fetch = mockFetch(
[],
textResponse(401, "invalid password"),
);
await expect(
assertWindmillBackendReachable(
buildSettings({ baseUrl: "http://backend.test/auth-failure" }),
),
).rejects.toThrow(
"Start a Windmill backend at that URL, or set WMILL_AI_EVAL_BACKEND_URL=<url>.",
);
});
});
describe("WindmillBackendClient", () => {
it("creates or reuses the specified backend workspace without deleting it", async () => {
const requests: Array<{ url: string; init?: RequestInit }> = [];
globalThis.fetch = mockFetch(
requests,
textResponse(200, "token"),
textResponse(200, "false"),
textResponse(200, ""),
);
const client = new WindmillBackendClient(
buildSettings({
baseUrl: "http://backend.test/shared-workspace",
workspaceOverride: "shared-evals",
}),
);
await expect(
client.withWorkspace("case-a", 1, async (workspaceId) => workspaceId),
).resolves.toBe("shared-evals");
expect(requests.map((entry) => entry.url)).toEqual([
"http://backend.test/shared-workspace/api/auth/login",
"http://backend.test/shared-workspace/api/workspaces/exists",
"http://backend.test/shared-workspace/api/workspaces/create",
]);
});
});
function buildSettings(
overrides: Partial<WindmillBackendSettings> = {},
): WindmillBackendSettings {
return {
baseUrl: "http://backend.test/default",
email: "admin@windmill.dev",
password: "changeme",
...overrides,
};
}
function mockFetch(
requests: Array<{ url: string; init?: RequestInit }>,
...responses: Response[]
): typeof fetch {
const queue = [...responses];
return async (input, init) => {
const url = String(input);
requests.push({ url, init });
const next = queue.shift();
if (!next) {
throw new Error(`Unexpected fetch: ${url}`);
}
return next;
};
}
function textResponse(status: number, body: string): Response {
return new Response(body, { status });
}
+23 -8
View File
@@ -3,6 +3,7 @@ import type { WindmillBackendSettings } from "../../core/windmillBackendSettings
const tokenCache = new Map<string, Promise<string>>();
const sharedWorkspaceQueue = new Map<string, Promise<void>>();
const DEFAULT_WORKSPACE_PREFIX = "ai-evals";
export class WindmillBackendClient {
constructor(private readonly settings: WindmillBackendSettings) {}
@@ -14,7 +15,7 @@ export class WindmillBackendClient {
): Promise<T> {
const workspaceId =
this.settings.workspaceOverride ??
buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt);
buildWorkspaceId(caseId, attempt);
const run = async () => {
await this.ensureWorkspace(workspaceId);
@@ -22,7 +23,7 @@ export class WindmillBackendClient {
try {
return await body(workspaceId);
} finally {
if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) {
if (!this.settings.workspaceOverride) {
await this.deleteWorkspace(workspaceId).catch(() => undefined);
}
}
@@ -136,6 +137,24 @@ export class WindmillBackendClient {
}
}
export async function assertWindmillBackendReachable(
settings: WindmillBackendSettings,
): Promise<void> {
try {
await new WindmillBackendClient(settings).getToken();
} catch (error) {
const details = error instanceof Error ? error.message : String(error);
throw new Error(
[
`Could not initialize the Windmill backend for AI eval proxy at ${settings.baseUrl}.`,
"Start a Windmill backend at that URL, or set WMILL_AI_EVAL_BACKEND_URL=<url>.",
`Using login ${settings.email}; if authentication failed, set WMILL_AI_EVAL_BACKEND_EMAIL and WMILL_AI_EVAL_BACKEND_PASSWORD.`,
`Details: ${details}`,
].join("\n"),
);
}
}
async function withSharedWorkspaceLock<T>(
workspaceId: string,
body: () => Promise<T>,
@@ -160,18 +179,14 @@ async function withSharedWorkspaceLock<T>(
}
}
function buildWorkspaceId(
prefix: string,
caseId: string,
attempt: number,
): string {
function buildWorkspaceId(caseId: string, attempt: number): string {
const caseSlug = caseId
.toLowerCase()
.replace(/[^a-z0-9-]+/g, "-")
.replace(/^-+|-+$/g, "")
.slice(0, 30);
const suffix = randomUUID().slice(0, 8);
return `${prefix}-${caseSlug || "case"}-a${attempt}-${suffix}`;
return `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || "case"}-a${attempt}-${suffix}`;
}
async function expectOk(response: Response, context: string): Promise<void> {
+89
View File
@@ -0,0 +1,89 @@
- id: global-test1-script-create
prompt: |-
Create a draft Bun script at `f/evals/global/greet_user`.
It should take a string `name` input and return `Hello, ${name}!`.
Leave it as an AI draft only; do not deploy or save it.
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/greet_user
language: bun
valueIncludes:
- name
- Hello
toolExpect:
requiredToolsUsed:
- write_script
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates a Bun script draft at f/evals/global/greet_user
- the script accepts a name input
- the script returns a greeting containing Hello, the provided name, and an exclamation mark
- the result stays as an AI draft and is not deployed or saved to the workspace
- id: global-test2-script-edit-existing
prompt: |-
Update the existing workspace script at `f/evals/global/format_greeting`.
Keep it as a Bun script, but change the greeting so the provided name is uppercased and the returned message ends with an exclamation mark.
Leave the result as an AI draft only; do not deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/format_greeting
language: bun
valueIncludes:
- toUpperCase
- "!"
toolExpect:
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates an AI draft for the existing f/evals/global/format_greeting script
- preserves the script as Bun
- uppercases the provided name in the greeting
- returns a message ending with an exclamation mark
- does not deploy or save the draft to the workspace
- id: global-test3-flow-create
prompt: |-
Create a draft flow at `f/evals/global/sum_numbers`.
It should take two numeric inputs, `a` and `b`, and return their sum.
Leave it as an AI draft only; do not deploy or save it.
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
path: f/evals/global/sum_numbers
valueIncludes:
- modules
- rawscript
- flow_input.a
- flow_input.b
toolExpect:
requiredToolsUsed:
- write_flow
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: write_flow
field: modules
stringStartsWithAnyOf:
- "["
judgeChecklist:
- creates a flow draft at f/evals/global/sum_numbers
- the flow accepts numeric inputs a and b
- the flow returns the sum of a and b
- the result stays as an AI draft and is not deployed or saved to the workspace
+9 -23
View File
@@ -27,11 +27,8 @@ import { EVAL_MODES, type EvalMode } from "../core/types";
import { DEFAULT_JUDGE_MODEL } from "../core/judge";
import { createCliModeRunner } from "../modes/cli";
import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime";
import {
FRONTEND_EVAL_TRANSPORTS,
type FrontendEvalTransport,
parseFrontendEvalTransport,
} from "../core/frontendTransport";
import { resolveWindmillBackendSettings } from "../core/windmillBackendSettings";
import { assertWindmillBackendReachable } from "../adapters/frontend/windmillBackend";
async function main() {
const program = new Command()
@@ -56,6 +53,7 @@ async function main() {
" bun run cli -- run flow --record",
" bun run cli -- run flow --backend-validation preview",
" bun run cli -- run flow flow-test5-simple-modification --runs 3",
" bun run cli -- run global global-test1-script-create",
" bun run cli -- run cli bun-hello-script",
"",
"Models:",
@@ -73,7 +71,7 @@ async function main() {
program
.command("cases")
.description("List available cases")
.argument("[mode]", "cli, flow, script, or app", parseOptionalMode)
.argument("[mode]", "cli, flow, script, app, or global", parseOptionalMode)
.action(async (mode?: EvalMode) => {
await handleCases(mode);
});
@@ -81,7 +79,7 @@ async function main() {
program
.command("run")
.description("Run one benchmark mode")
.argument("<mode>", "cli, flow, script, or app", parseMode)
.argument("<mode>", "cli, flow, script, app, or global", parseMode)
.argument("[caseIds...]", "specific case ids to run")
.option(
"--runs <n>",
@@ -98,10 +96,6 @@ async function main() {
"--models <names>",
"comma-separated model aliases to run sequentially",
)
.option(
"--transport <mode>",
`frontend transport (${FRONTEND_EVAL_TRANSPORTS.join(", ")})`,
)
.option("--verbose", "stream assistant output during frontend runs")
.option(
"--record",
@@ -120,7 +114,6 @@ async function main() {
output?: string;
model?: string;
models?: string;
transport?: string;
verbose?: boolean;
record?: boolean;
backendValidation?: string;
@@ -133,9 +126,6 @@ async function main() {
outputPath: options.output,
model: options.model,
models: options.models,
transport: options.transport
? parseFrontendEvalTransport(options.transport)
: undefined,
verbose: options.verbose ?? false,
record: options.record ?? false,
backendValidation: options.backendValidation,
@@ -163,7 +153,7 @@ function handleModels() {
process.stdout.write("Available models\n");
for (const model of EVAL_MODELS) {
const supports = [
...(model.frontend ? ["flow", "script", "app"] : []),
...(model.frontend ? ["flow", "script", "app", "global"] : []),
...(model.cli ? ["cli"] : []),
];
const aliases = [
@@ -184,7 +174,6 @@ async function handleRun(input: {
outputPath?: string;
model?: string;
models?: string;
transport?: FrontendEvalTransport;
verbose: boolean;
record: boolean;
backendValidation?: string;
@@ -197,11 +186,6 @@ async function handleRun(input: {
if (input.model && input.models) {
throw new Error("Use either --model or --models, not both");
}
if (input.mode === "cli" && input.transport === "proxy") {
throw new Error(
"--transport proxy is only supported for flow, script, and app modes",
);
}
const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
const models = resolveRequestedModels(input.mode, input.model, input.models);
@@ -220,6 +204,9 @@ async function handleRun(input: {
"--backend-validation currently supports only flow and script modes",
);
}
if (input.mode !== "cli") {
await assertWindmillBackendReachable(resolveWindmillBackendSettings());
}
const summaries: Array<{
label: string;
@@ -249,7 +236,6 @@ async function handleRun(input: {
caseIds: input.caseIds,
runs: input.runs,
model: model.id,
transport: input.transport,
verbose: input.verbose,
backendValidation,
});
-2
View File
@@ -13,9 +13,7 @@ export interface BackendValidationSettings {
baseUrl: string;
email: string;
password: string;
keepWorkspaces: boolean;
workspaceOverride?: string;
workspacePrefix: string;
pollIntervalMs: number;
maxWaitMs: number;
}
+20
View File
@@ -183,6 +183,26 @@ describe("loadCases", () => {
});
});
it("loads global draft validation and forbidden tool expectations", async () => {
const globalCases = await loadCases("global");
const caseEntry = globalCases.find((entry) => entry.id === "global-test1-script-create");
expect(caseEntry?.validate).toMatchObject({
draftCountExactly: 1,
requiredDrafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
},
],
});
expect(caseEntry?.toolExpect).toMatchObject({
requiredToolsUsed: ["write_script"],
forbiddenToolsUsed: ["deploy_workspace_item", "delete_workspace_item"],
});
});
it("loads tool expectations for workspace mutation cases", async () => {
const scriptCases = await loadCases("script");
const caseEntry = scriptCases.find(
-64
View File
@@ -1,64 +0,0 @@
import { afterEach, describe, expect, it } from "bun:test";
import {
parseFrontendEvalTransport,
resolveFrontendEvalTransportSettings,
} from "./frontendTransport";
const ORIGINAL_ENV = {
WMILL_AI_EVAL_BACKEND_URL: process.env.WMILL_AI_EVAL_BACKEND_URL,
};
afterEach(() => {
if (ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL === undefined) {
delete process.env.WMILL_AI_EVAL_BACKEND_URL;
} else {
process.env.WMILL_AI_EVAL_BACKEND_URL =
ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL;
}
});
describe("parseFrontendEvalTransport", () => {
it("defaults to direct when unset", () => {
expect(parseFrontendEvalTransport(undefined)).toBe("direct");
});
it("accepts proxy explicitly", () => {
expect(parseFrontendEvalTransport("proxy")).toBe("proxy");
});
it("rejects unsupported values", () => {
expect(() => parseFrontendEvalTransport("worker")).toThrow(
"Unsupported frontend eval transport: worker",
);
});
});
describe("resolveFrontendEvalTransportSettings", () => {
it("includes backend settings for proxy transport", () => {
process.env.WMILL_AI_EVAL_BACKEND_URL = "http://127.0.0.1:8000/";
expect(
resolveFrontendEvalTransportSettings({
evalMode: "app",
requestedTransport: "proxy",
}),
).toMatchObject({
transport: "proxy",
backend: {
baseUrl: "http://127.0.0.1:8000",
},
});
});
it("keeps direct transport for cli runs", () => {
expect(
resolveFrontendEvalTransportSettings({
evalMode: "cli",
requestedTransport: "direct",
}),
).toEqual({
transport: "direct",
backend: undefined,
});
});
});
-49
View File
@@ -1,49 +0,0 @@
import type { EvalMode } from "./types";
import type { WindmillBackendSettings } from "./windmillBackendSettings";
import { resolveWindmillBackendSettings } from "./windmillBackendSettings";
export const FRONTEND_EVAL_TRANSPORTS = ["direct", "proxy"] as const;
export type FrontendEvalTransport = (typeof FRONTEND_EVAL_TRANSPORTS)[number];
export interface FrontendEvalTransportSettings {
transport: FrontendEvalTransport;
backend?: WindmillBackendSettings;
}
export function parseFrontendEvalTransport(
value?: string | null,
): FrontendEvalTransport {
const normalized = value?.trim().toLowerCase();
if (!normalized || normalized === "direct") {
return "direct";
}
if (normalized === "proxy") {
return "proxy";
}
throw new Error(
`Unsupported frontend eval transport: ${value}. Use one of: ${FRONTEND_EVAL_TRANSPORTS.join(", ")}`,
);
}
export function resolveFrontendEvalTransportSettings(input: {
evalMode: EvalMode;
requestedTransport?: string | null;
}): FrontendEvalTransportSettings {
const transport = parseFrontendEvalTransport(input.requestedTransport);
if (transport === "proxy" && input.evalMode === "cli") {
throw new Error(
'Frontend eval transport "proxy" is only supported for flow, script, and app evals',
);
}
return {
transport,
backend:
transport === "proxy" ? resolveWindmillBackendSettings() : undefined,
};
}
+1 -1
View File
@@ -145,7 +145,7 @@ export function resolveEvalModel(mode: EvalMode, alias?: string): EvalModelSpec
export function getEvalModelHelpText(): string {
return EVAL_MODELS.map((model) => {
const modes = [
...(model.frontend ? ["flow", "script", "app"] : []),
...(model.frontend ? ["flow", "script", "app", "global"] : []),
...(model.cli ? ["cli"] : []),
];
return ` ${model.id.padEnd(8)} ${model.label} (${modes.join(", ")})`;
-6
View File
@@ -74,7 +74,6 @@ export function buildRunResult(input: {
mode: EvalMode;
runs: number;
runModel: string | null;
transport?: BenchmarkRunResult["transport"];
judgeModel: string | null;
caseResults: BenchmarkCaseResult[];
}): BenchmarkRunResult {
@@ -116,7 +115,6 @@ export function buildRunResult(input: {
gitSha: getGitSha(),
runs: input.runs,
runModel: input.runModel,
transport: input.transport ?? null,
judgeModel: input.judgeModel,
caseCount: input.caseResults.length,
attemptCount,
@@ -142,9 +140,6 @@ export function formatRunSummary(result: BenchmarkRunResult): string {
`Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
`Average duration: ${Math.round(result.averageDurationMs)}ms`,
];
if (result.transport) {
lines.splice(1, 0, `Transport: ${result.transport}`);
}
const failures = collectFailures(result);
if (failures.length > 0) {
@@ -251,7 +246,6 @@ function toHistoryRecord(result: BenchmarkRunResult) {
mode: result.mode,
runs: result.runs,
runModel: result.runModel,
transport: result.transport,
judgeModel: result.judgeModel,
caseCount: result.caseCount,
attemptCount: result.attemptCount,
+24 -4
View File
@@ -1,7 +1,6 @@
export const EVAL_MODES = ["cli", "flow", "script", "app"] as const;
export const EVAL_MODES = ["cli", "flow", "script", "app", "global"] as const;
export type EvalMode = (typeof EVAL_MODES)[number];
export type FrontendEvalTransport = "direct" | "proxy";
export interface EvalCaseRuntimeBackendPreview {
args?: Record<string, unknown>;
@@ -109,6 +108,27 @@ export interface AppValidationSpec {
forbiddenAppContent?: string[];
}
export interface GlobalDraftRequirement {
type: string;
path: string;
triggerKind?: string;
language?: string;
summaryIncludes?: string[];
valueIncludes?: string[];
valueExcludes?: string[];
}
export interface GlobalValidationSpec {
draftCountAtLeast?: number;
draftCountExactly?: number;
requiredDrafts?: GlobalDraftRequirement[];
forbiddenDrafts?: Array<{
type: string;
path: string;
triggerKind?: string;
}>;
}
export interface CliValidationSpec {
requiredSkills?: string[];
forbiddenSkills?: string[];
@@ -137,10 +157,11 @@ export interface ToolCallArgumentRule {
export interface ToolValidationSpec {
requiredToolsUsed?: string[];
forbiddenToolsUsed?: string[];
toolCallArgs?: ToolCallArgumentRule[];
}
export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec;
export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec | GlobalValidationSpec;
export interface EvalCase {
id: string;
@@ -297,7 +318,6 @@ export interface BenchmarkRunResult {
gitSha: string | null;
runs: number;
runModel: string | null;
transport: FrontendEvalTransport | null;
judgeModel: string | null;
caseCount: number;
attemptCount: number;
+225
View File
@@ -2,6 +2,7 @@ import { describe, expect, it } from "bun:test";
import {
validateAppState,
validateCliWorkspace,
validateGlobalState,
validateScriptState,
validateToolExpectations,
} from "./validators";
@@ -117,6 +118,230 @@ describe("validateToolExpectations", () => {
details: 'rejected prefixes: schedules/; values: "schedules/greet_user_daily"',
});
});
it("rejects forbidden tool usage", () => {
const checks = validateToolExpectations({
run: {
success: true,
actual: {},
assistantMessageCount: 1,
toolCallCount: 1,
toolsUsed: ["write_script", "deploy_workspace_item"],
skillsInvoked: [],
},
toolExpect: {
forbiddenToolsUsed: ["deploy_workspace_item"],
},
});
expect(checks).toContainEqual({
name: "does not use deploy_workspace_item",
passed: false,
details: "tools used: write_script, deploy_workspace_item",
});
});
});
describe("validateGlobalState", () => {
it("accepts a required script draft", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
validate: {
draftCountExactly: 1,
requiredDrafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
valueIncludes: ["Hello"],
},
],
},
});
expect(checks.every((check) => check.passed)).toBe(true);
});
it("fails when a required draft is missing", () => {
const checks = validateGlobalState({
actual: {
drafts: [],
},
validate: {
requiredDrafts: [
{
type: "script",
path: "f/evals/global/greet_user",
},
],
},
});
expect(checks).toContainEqual({
name: "global includes script draft f/evals/global/greet_user",
passed: false,
details: "drafts: none",
});
});
it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_python",
language: "python3",
value: "def main(name: str):\n return f'Hello, {name}!'\n",
isDraft: true,
},
],
},
});
expect(checks.some((check) => check.name.includes("exports entrypoint"))).toBe(
false
);
expect(checks.every((check) => check.passed)).toBe(true);
});
it("allows read-only global cases without draft expectations", () => {
const checks = validateGlobalState({
actual: {
drafts: [],
},
});
expect(
checks.some(
(check) => check.name === "global produced at least one draft"
)
).toBe(false);
expect(checks.every((check) => check.passed)).toBe(true);
});
it("matches expected global draft fixtures", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
value:
"export async function main(name: string) {\r\n return `Hello, ${name}!`\r\n}\r\n",
isDraft: true,
},
],
},
expected: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
});
expect(checks).toContainEqual({
name: "global drafts match expected",
passed: true,
});
});
it("fails when expected global draft fixtures differ", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
expected: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
value:
"export async function main(name: string) {\n return `Bonjour, ${name}!`\n}\n",
isDraft: true,
},
],
},
});
const expectedMatchCheck = checks.find(
(check) => check.name === "global drafts match expected"
);
expect(expectedMatchCheck?.passed).toBe(false);
expect(expectedMatchCheck?.details).toContain(
"script:f/evals/global/greet_user value differs"
);
expect(expectedMatchCheck?.details).toContain("Hello");
expect(expectedMatchCheck?.details).toContain("Bonjour");
});
it("explains expected global draft metadata mismatches", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "bun",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
expected: {
drafts: [
{
type: "script",
path: "f/evals/global/greet_user",
language: "python3",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
});
const expectedMatchCheck = checks.find(
(check) => check.name === "global drafts match expected"
);
expect(expectedMatchCheck?.passed).toBe(false);
expect(expectedMatchCheck?.details).toContain(
"script:f/evals/global/greet_user language differs"
);
expect(expectedMatchCheck?.details).toContain('actual="bun"');
expect(expectedMatchCheck?.details).toContain('expected="python3"');
});
});
describe("validateAppState", () => {
+376
View File
@@ -6,6 +6,7 @@ import type {
CliTrace,
CliValidationSpec,
FlowValidationSpec,
GlobalValidationSpec,
ModeRunOutput,
ToolValidationSpec,
} from "./types";
@@ -51,6 +52,20 @@ export interface AppDatatableState {
error?: string;
}
export interface GlobalDraftState {
drafts: GlobalDraft[];
}
export interface GlobalDraft {
type: string;
path: string;
triggerKind?: string;
summary?: string;
language?: string;
value?: unknown;
isDraft?: boolean;
}
const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]);
const CONTROL_FLOW_MODULE_TYPES = new Set(["branchone", "branchall", "forloopflow", "whileloopflow"]);
@@ -154,6 +169,16 @@ export function validateToolExpectations(input: {
);
}
for (const toolName of expect.forbiddenToolsUsed ?? []) {
checks.push(
check(
`does not use ${toolName}`,
!input.run.toolsUsed.includes(toolName),
`tools used: ${input.run.toolsUsed.join(", ") || "none"}`
)
);
}
for (const rule of expect.toolCallArgs ?? []) {
const calls = toolCallDetails.filter((call) => call.name === rule.tool);
checks.push(
@@ -202,6 +227,161 @@ export function validateToolExpectations(input: {
return checks;
}
export function validateGlobalState(input: {
actual: GlobalDraftState;
expected?: GlobalDraftState;
validate?: GlobalValidationSpec;
}): BenchmarkCheck[] {
const drafts = input.actual.drafts ?? [];
const checks: BenchmarkCheck[] = [];
// Read-only global cases are valid; only enforce draft production when the
// case explicitly asks for draft output.
if (globalValidationExpectsDrafts(input)) {
checks.push(
check(
"global produced at least one draft",
drafts.length > 0,
`drafts=${drafts.length}`
)
);
}
checks.push(
check(
"all global outputs are drafts",
drafts.every((draft) => draft.isDraft === true),
summarizeGlobalDrafts(drafts)
)
);
for (const draft of drafts) {
if (draft.type !== "script" || typeof draft.value !== "string") {
continue;
}
const language = (draft.language ?? "bun").toLowerCase();
const syntaxErrors = getScriptSyntaxErrors(draft.value, language);
if (TS_LIKE_LANGUAGES.has(language)) {
checks.push(
check(
`script draft ${draft.path} exports entrypoint`,
hasSupportedEntrypoint(draft.value)
)
);
}
checks.push(
check(
`script draft ${draft.path} has no syntax errors`,
syntaxErrors.length === 0,
summarizeProblems(syntaxErrors)
)
);
}
if (input.expected) {
checks.push(
check(
"global drafts match expected",
globalDraftStatesEqual(input.actual, input.expected),
describeGlobalDraftStateMismatch(input.actual, input.expected)
)
);
}
const validate = input.validate;
if (!validate) {
return checks;
}
if (validate.draftCountAtLeast !== undefined) {
checks.push(
check(
`global includes at least ${validate.draftCountAtLeast} draft(s)`,
drafts.length >= validate.draftCountAtLeast,
`drafts=${drafts.length}`
)
);
}
if (validate.draftCountExactly !== undefined) {
checks.push(
check(
`global includes exactly ${validate.draftCountExactly} draft(s)`,
drafts.length === validate.draftCountExactly,
`drafts=${drafts.length}`
)
);
}
for (const required of validate.requiredDrafts ?? []) {
const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind);
checks.push(
check(
`global includes ${required.type} draft ${required.path}`,
Boolean(draft),
summarizeGlobalDrafts(drafts)
)
);
if (!draft) {
continue;
}
if (required.language !== undefined) {
checks.push(
check(
`${required.type} draft ${required.path} uses ${required.language}`,
draft.language === required.language,
`language=${draft.language ?? "(none)"}`
)
);
}
for (const snippet of required.summaryIncludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} summary includes '${snippet}'`,
normalizeText(draft.summary ?? "").includes(normalizeText(snippet)),
`summary=${draft.summary ?? ""}`
)
);
}
const valueText = stringifyGlobalDraftValue(draft.value);
for (const snippet of required.valueIncludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} value includes '${snippet}'`,
normalizeText(valueText).includes(normalizeText(snippet)),
truncateForDetails(valueText)
)
);
}
for (const snippet of required.valueExcludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} value excludes '${snippet}'`,
!normalizeText(valueText).includes(normalizeText(snippet)),
truncateForDetails(valueText)
)
);
}
}
for (const forbidden of validate.forbiddenDrafts ?? []) {
checks.push(
check(
`global does not include ${forbidden.type} draft ${forbidden.path}`,
!findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind),
summarizeGlobalDrafts(drafts)
)
);
}
return checks;
}
export function validateAppState(input: {
actual: AppFilesState;
initial?: AppFilesState;
@@ -433,6 +613,202 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined {
return `${problems.slice(0, limit).join("; ")}; ...and ${problems.length - limit} more`;
}
function findGlobalDraft(
drafts: GlobalDraft[],
type: string,
path: string,
triggerKind?: string
): GlobalDraft | undefined {
return drafts.find(
(draft) =>
draft.type === type &&
draft.path === path &&
(triggerKind === undefined || draft.triggerKind === triggerKind)
);
}
function summarizeGlobalDrafts(drafts: GlobalDraft[]): string {
const summary = drafts
.map((draft) => formatGlobalDraftKey(draft))
.join(", ");
return `drafts: ${summary || "none"}`;
}
function formatGlobalDraftKey(draft: GlobalDraft): string {
return `${draft.type}${draft.triggerKind ? `:${draft.triggerKind}` : ""}:${draft.path}`;
}
function globalValidationExpectsDrafts(input: {
expected?: GlobalDraftState;
validate?: GlobalValidationSpec;
}): boolean {
const validate = input.validate;
return (
(input.expected?.drafts?.length ?? 0) > 0 ||
(validate?.requiredDrafts?.length ?? 0) > 0 ||
(validate?.draftCountAtLeast ?? 0) > 0 ||
(validate?.draftCountExactly ?? 0) > 0
);
}
function globalDraftStatesEqual(left: GlobalDraftState, right: GlobalDraftState): boolean {
return (
JSON.stringify(canonicalizeGlobalDrafts(left.drafts ?? [])) ===
JSON.stringify(canonicalizeGlobalDrafts(right.drafts ?? []))
);
}
function describeGlobalDraftStateMismatch(
actual: GlobalDraftState,
expected: GlobalDraftState
): string {
const actualDrafts = actual.drafts ?? [];
const expectedDrafts = expected.drafts ?? [];
const actualByKey = new Map(
actualDrafts.map((draft) => [globalDraftSortKey(draft), draft] as const)
);
const expectedByKey = new Map(
expectedDrafts.map((draft) => [globalDraftSortKey(draft), draft] as const)
);
for (const key of Array.from(expectedByKey.keys()).sort()) {
const expectedDraft = expectedByKey.get(key);
if (expectedDraft && !actualByKey.has(key)) {
return `missing expected draft ${formatGlobalDraftKey(expectedDraft)}; actual=${summarizeGlobalDrafts(actualDrafts)}`;
}
}
for (const key of Array.from(actualByKey.keys()).sort()) {
const actualDraft = actualByKey.get(key);
if (actualDraft && !expectedByKey.has(key)) {
return `unexpected draft ${formatGlobalDraftKey(actualDraft)}; expected=${summarizeGlobalDrafts(expectedDrafts)}`;
}
}
for (const key of Array.from(expectedByKey.keys()).sort()) {
const actualDraft = actualByKey.get(key);
const expectedDraft = expectedByKey.get(key);
if (!actualDraft || !expectedDraft) {
continue;
}
const fieldMismatch = describeGlobalDraftFieldMismatch(
formatGlobalDraftKey(expectedDraft),
actualDraft,
expectedDraft
);
if (fieldMismatch) {
return fieldMismatch;
}
}
return `actual=${summarizeGlobalDrafts(actualDrafts)}; expected=${summarizeGlobalDrafts(expectedDrafts)}`;
}
function describeGlobalDraftFieldMismatch(
key: string,
actual: GlobalDraft,
expected: GlobalDraft
): string | undefined {
const fields: Array<"language" | "summary" | "value" | "isDraft"> = [
"language",
"summary",
"value",
"isDraft",
];
for (const field of fields) {
const actualValue = comparableGlobalDraftFieldValue(actual, field);
const expectedValue = comparableGlobalDraftFieldValue(expected, field);
if (JSON.stringify(actualValue) === JSON.stringify(expectedValue)) {
continue;
}
return `${key} ${field} differs: actual=${formatGlobalDraftFieldValue(
actualValue
)}; expected=${formatGlobalDraftFieldValue(expectedValue)}`;
}
return undefined;
}
function comparableGlobalDraftFieldValue(
draft: GlobalDraft,
field: "language" | "summary" | "value" | "isDraft"
): unknown {
if (field === "summary" && typeof draft.summary === "string") {
return normalizeText(draft.summary);
}
if (field === "value" && typeof draft.value === "string") {
return normalizeText(draft.value);
}
if (field === "value") {
return canonicalizeJsonValue(draft.value);
}
return draft[field];
}
function formatGlobalDraftFieldValue(value: unknown): string {
if (value === undefined) {
return "(missing)";
}
return truncateForDetails(JSON.stringify(value), 300);
}
function canonicalizeGlobalDrafts(drafts: GlobalDraft[]): unknown[] {
return drafts
.slice()
.sort((left, right) => globalDraftSortKey(left).localeCompare(globalDraftSortKey(right)))
.map((draft) =>
canonicalizeJsonValue({
type: draft.type,
path: draft.path,
triggerKind: draft.triggerKind,
language: draft.language,
summary:
typeof draft.summary === "string" ? normalizeText(draft.summary) : draft.summary,
value:
typeof draft.value === "string"
? normalizeText(draft.value)
: canonicalizeJsonValue(draft.value),
isDraft: draft.isDraft,
})
);
}
function globalDraftSortKey(draft: GlobalDraft): string {
return `${draft.type}:${draft.triggerKind ?? ""}:${draft.path}`;
}
function canonicalizeJsonValue(value: unknown): unknown {
if (Array.isArray(value)) {
return value.map(canonicalizeJsonValue);
}
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.sort(([left], [right]) => left.localeCompare(right))
.map(([key, nested]) => [key, canonicalizeJsonValue(nested)])
);
}
return value;
}
function stringifyGlobalDraftValue(value: unknown): string {
if (typeof value === "string") {
return value;
}
return JSON.stringify(value ?? null, null, 2);
}
function truncateForDetails(value: string, maxLength = 500): string {
const normalized = value.replace(/\s+/g, " ").trim();
if (normalized.length <= maxLength) {
return normalized;
}
return `${normalized.slice(0, Math.max(0, maxLength - 3))}...`;
}
function validateCliExpectations(
assistantOutput: string,
trace: CliTrace | undefined,
@@ -0,0 +1,63 @@
import { afterEach, describe, expect, it } from "bun:test";
import { resolveWindmillBackendSettings } from "./windmillBackendSettings";
const ENV_KEYS = [
"WMILL_AI_EVAL_BACKEND_URL",
"WINDMILL_URL",
"WINDMILL_BASE_URL",
"REMOTE",
"WMILL_AI_EVAL_BACKEND_EMAIL",
"WMILL_AI_EVAL_BACKEND_PASSWORD",
"WMILL_AI_EVAL_BACKEND_WORKSPACE",
] as const;
const ORIGINAL_ENV = Object.fromEntries(
ENV_KEYS.map((key) => [key, process.env[key]]),
) as Record<(typeof ENV_KEYS)[number], string | undefined>;
afterEach(() => {
for (const key of ENV_KEYS) {
const value = ORIGINAL_ENV[key];
if (value === undefined) {
delete process.env[key];
} else {
process.env[key] = value;
}
}
});
describe("resolveWindmillBackendSettings", () => {
it("uses backend URL/auth defaults and the optional explicit workspace", () => {
delete process.env.WMILL_AI_EVAL_BACKEND_URL;
delete process.env.WINDMILL_URL;
delete process.env.WINDMILL_BASE_URL;
delete process.env.REMOTE;
process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE = "shared-evals";
expect(resolveWindmillBackendSettings()).toEqual({
baseUrl: "http://127.0.0.1:8000",
email: "admin@windmill.dev",
password: "changeme",
workspaceOverride: "shared-evals",
});
});
it("does not expose workspace retention knobs", () => {
process.env.WMILL_AI_EVAL_BACKEND_URL = "http://backend.test/";
const settings = resolveWindmillBackendSettings();
expect(settings).toEqual({
baseUrl: "http://backend.test",
email: "admin@windmill.dev",
password: "changeme",
workspaceOverride: undefined,
});
expect(Object.keys(settings).sort()).toEqual([
"baseUrl",
"email",
"password",
"workspaceOverride",
]);
});
});
-22
View File
@@ -2,9 +2,7 @@ export interface WindmillBackendSettings {
baseUrl: string;
email: string;
password: string;
keepWorkspaces: boolean;
workspaceOverride?: string;
workspacePrefix: string;
}
export function resolveWindmillBackendSettings(): WindmillBackendSettings {
@@ -18,13 +16,9 @@ export function resolveWindmillBackendSettings(): WindmillBackendSettings {
),
email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev",
password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme",
keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES),
workspaceOverride: sanitizeOptionalWorkspaceId(
process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE,
),
workspacePrefix: sanitizeWorkspacePrefix(
process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals",
),
};
}
@@ -43,25 +37,9 @@ function normalizeBaseUrl(value: string): string {
return value.replace(/\/+$/, "");
}
function sanitizeWorkspacePrefix(value: string): string {
const sanitized = value
.trim()
.toLowerCase()
.replace(/[^a-z0-9-]+/g, "-")
.replace(/^-+|-+$/g, "");
return sanitized.length > 0 ? sanitized : "ai-evals";
}
function sanitizeOptionalWorkspaceId(
value: string | undefined,
): string | undefined {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
}
function isTruthy(value: string | undefined): boolean {
if (!value) {
return false;
}
return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase());
}
@@ -0,0 +1,23 @@
{
"workspace": {
"scripts": [
{
"path": "f/evals/global/format_greeting",
"summary": "Format a greeting for a provided name",
"description": "Returns a plain greeting for the provided name.",
"language": "bun",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"required": ["name"]
},
"content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n"
}
]
}
}
+5 -9
View File
@@ -5,15 +5,12 @@ import type { FrontendEvalModelConfig } from "../core/models";
import { validateAppState, type AppFilesState } from "../core/validators";
import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
import { runAppEval } from "../adapters/frontend/core/app/appEvalRunner";
import {
DEFAULT_FRONTEND_EVAL_MODEL,
getFrontendApiKey,
} from "./frontendCommon";
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
import { getFrontendApiKey } from "./frontendCommon";
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
export function createAppModeRunner(
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
transportSettings?: FrontendEvalTransportSettings,
modelConfig: FrontendEvalModelConfig,
backendSettings: WindmillBackendSettings,
): ModeRunner<AppFilesState, AppFilesState, AppFilesState> {
return {
mode: "app",
@@ -37,8 +34,7 @@ export function createAppModeRunner(
appContext: context.evalCase?.runtime?.appContext,
provider: modelConfig.provider,
model: modelConfig.model,
transport: transportSettings?.transport,
backend: transportSettings?.backend,
backend: backendSettings,
runContext: context,
},
);
+6 -10
View File
@@ -7,11 +7,8 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
import { runFlowEval } from "../adapters/frontend/core/flow/flowEvalRunner";
import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers";
import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
import {
DEFAULT_FRONTEND_EVAL_MODEL,
getFrontendApiKey,
} from "./frontendCommon";
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
import { getFrontendApiKey } from "./frontendCommon";
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
import {
normalizeFlowInitialFixture,
normalizeFlowStateFixture,
@@ -19,9 +16,9 @@ import {
} from "./flowFixtures";
export function createFlowModeRunner(
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
backendValidation?: BackendValidationSettings,
transportSettings?: FrontendEvalTransportSettings,
modelConfig: FrontendEvalModelConfig,
backendValidation: BackendValidationSettings | undefined,
backendSettings: WindmillBackendSettings,
): ModeRunner<FlowInitialFixture, FlowState, FlowState> {
return {
mode: "flow",
@@ -49,8 +46,7 @@ export function createFlowModeRunner(
maxIterations: context.evalCase?.runtime?.maxTurns,
provider: modelConfig.provider,
model: modelConfig.model,
transport: transportSettings?.transport,
backend: transportSettings?.backend,
backend: backendSettings,
runContext: context,
},
);
+1 -9
View File
@@ -1,12 +1,4 @@
import {
getFrontendEvalModel,
resolveEvalModel,
type FrontendEvalModelConfig,
} from "../core/models";
export const DEFAULT_FRONTEND_EVAL_MODEL: FrontendEvalModelConfig = getFrontendEvalModel(
resolveEvalModel("flow")
);
import type { FrontendEvalModelConfig } from "../core/models";
export function getFrontendApiKey(provider: FrontendEvalModelConfig["provider"]): string {
const envName =
+81
View File
@@ -0,0 +1,81 @@
import { readFile } from "node:fs/promises";
import { runGlobalEval } from "../adapters/frontend/core/global/globalEvalRunner";
import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend";
import type { FrontendEvalModelConfig } from "../core/models";
import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types";
import { validateGlobalState, type GlobalDraftState } from "../core/validators";
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
import { getFrontendApiKey } from "./frontendCommon";
export interface GlobalInitialFixture {
workspace?: BenchmarkWorkspaceRunnables;
}
export function createGlobalModeRunner(
modelConfig: FrontendEvalModelConfig,
backendSettings: WindmillBackendSettings,
): ModeRunner<GlobalInitialFixture, GlobalDraftState, GlobalDraftState> {
return {
mode: "global",
concurrency: 3,
judgeThreshold: 80,
async loadInitial(path) {
return path ? await loadGlobalInitialFixture(path) : undefined;
},
async loadExpected(path) {
return path ? await loadGlobalExpectedFixture(path) : undefined;
},
async run(prompt, initial, context) {
const result = await runGlobalEval(
prompt,
getFrontendApiKey(modelConfig.provider),
{
workspaceFixtures: initial?.workspace,
maxIterations: context.evalCase?.runtime?.maxTurns,
provider: modelConfig.provider,
model: modelConfig.model,
backend: backendSettings,
runContext: context,
},
);
return {
success: result.success,
actual: result.state,
error: result.error,
assistantMessageCount: result.assistantMessageCount,
toolCallCount: result.toolCallCount,
toolsUsed: result.toolsUsed,
toolCallDetails: result.toolCallDetails,
skillsInvoked: [],
tokenUsage: result.tokenUsage,
};
},
validate({ evalCase, actual, expected }) {
return validateGlobalState({
actual,
expected,
validate: evalCase.validate as GlobalValidationSpec | undefined,
});
},
buildArtifacts(actual): BenchmarkArtifactFile[] {
return [
{
path: "global-drafts.json",
content: JSON.stringify(actual, null, 2) + "\n",
},
];
},
};
}
async function loadGlobalInitialFixture(path: string): Promise<GlobalInitialFixture> {
const parsed = JSON.parse(await readFile(path, "utf8")) as GlobalInitialFixture;
return {
workspace: parsed.workspace ?? {},
};
}
async function loadGlobalExpectedFixture(path: string): Promise<GlobalDraftState> {
return JSON.parse(await readFile(path, "utf8")) as GlobalDraftState;
}
+6 -10
View File
@@ -6,16 +6,13 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner";
import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers";
import {
DEFAULT_FRONTEND_EVAL_MODEL,
getFrontendApiKey,
} from "./frontendCommon";
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
import { getFrontendApiKey } from "./frontendCommon";
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
export function createScriptModeRunner(
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
backendValidation?: BackendValidationSettings,
transportSettings?: FrontendEvalTransportSettings,
modelConfig: FrontendEvalModelConfig,
backendValidation: BackendValidationSettings | undefined,
backendSettings: WindmillBackendSettings,
): ModeRunner<ScriptEvalState, ScriptEvalState, ScriptEvalState> {
return {
mode: "script",
@@ -40,8 +37,7 @@ export function createScriptModeRunner(
maxIterations: context.evalCase?.runtime?.maxTurns,
provider: modelConfig.provider,
model: modelConfig.model,
transport: transportSettings?.transport,
backend: transportSettings?.backend,
backend: backendSettings,
runContext: context,
},
);
@@ -0,0 +1,94 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled, NULL::text as workspace_id FROM password WHERE email = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "devops",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "verified",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "first_time_user",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "role_source",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "disabled",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null,
false,
false,
false,
true,
true,
true,
null,
false,
false,
false,
null
]
},
"hash": "0142d9dc9c1b57487dd5709a0376794f18d33e5bd6340c0189be7818cda64328"
}
@@ -0,0 +1,95 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email as \"email!\", login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, NULL::bool as operator_only, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id FROM password\n UNION ALL\n SELECT email as \"email!\", 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, true as operator_only, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC, \"email!\"\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "verified!",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "super_admin!",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "devops!",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "first_time_user!",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "role_source!",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "disabled!",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
]
},
"hash": "16b4496c21d0619dab4521dca22e5fe144c59156a8f06d8592291684c49b2f37"
}
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4"
}
@@ -0,0 +1,66 @@
{
"db_name": "PostgreSQL",
"query": "SELECT label, token_prefix, expiration, created_at, last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "label",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "token_prefix",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "expiration",
"type_info": "Timestamptz"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "last_used_at",
"type_info": "Timestamptz"
},
{
"ordinal": 5,
"name": "scopes",
"type_info": "TextArray"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "read_only",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
true,
false,
true,
false,
false,
true,
true,
false
]
},
"hash": "52777947d60d5ddd6d28852a1cf104a5af2fd565706204b7d5d33594ff91e61e"
}
@@ -15,7 +15,7 @@
]
},
"nullable": [
true
null
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO token\n (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id, read_only)\n SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10\n WHERE $9::varchar IS NULL OR NOT EXISTS(\n SELECT 1 FROM workspace WHERE id = $9 AND deleted = true\n )",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Timestamptz",
"Bool",
"TextArray",
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "9a1bf7c424154172b56bbd29b51ef468969b5452150ee6fffe31d59b52a4f1c2"
}
@@ -0,0 +1,95 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n UNION ALL\n SELECT email as \"email!\", true as operator_only, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "verified!",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "super_admin!",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "devops!",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "first_time_user!",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "role_source!",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "disabled!",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
]
},
"hash": "c17c39add3f70218dbae38595a909d02cfabe7e0864af577df6968428ac448ef"
}
@@ -0,0 +1,66 @@
{
"db_name": "PostgreSQL",
"query": "SELECT label, token_prefix, expiration, created_at, last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "label",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "token_prefix",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "expiration",
"type_info": "Timestamptz"
},
{
"ordinal": 3,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "last_used_at",
"type_info": "Timestamptz"
},
{
"ordinal": 5,
"name": "scopes",
"type_info": "TextArray"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "read_only",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Int8",
"Int8"
]
},
"nullable": [
true,
false,
true,
false,
false,
true,
true,
false
]
},
"hash": "e001fb68c60fa736cecce52be49c9fb0714c9c91a7ea3924811d345bee94c013"
}
@@ -0,0 +1,53 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE token SET last_used_at = now() WHERE\n token_hash = $1\n AND (expiration > NOW() OR expiration IS NULL)\n AND (workspace_id IS NULL OR workspace_id = $2)\n RETURNING owner, email, super_admin, scopes, label, read_only",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "owner",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "scopes",
"type_info": "TextArray"
},
{
"ordinal": 4,
"name": "label",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "read_only",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
true,
false,
true,
true,
false
]
},
"hash": "ed82ffc4d806a19519701e39abc43e8e250cc9772cca4a7dae3442be86f4060c"
}
+296 -282
View File
File diff suppressed because it is too large Load Diff
+2 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.699.0"
version = "1.702.1"
authors.workspace = true
edition.workspace = true
@@ -87,7 +87,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.699.0"
version = "1.702.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
+1 -1
View File
@@ -1 +1 @@
c6cd1afe2d9e04809b30751cd1687b28a65e62b1
19a76a09ffb43649ee19e62d07e8b8a42d78757b
@@ -0,0 +1 @@
ALTER TABLE token DROP COLUMN IF EXISTS read_only;
@@ -0,0 +1,4 @@
-- Add a flag to restrict a token to read-only HTTP endpoints.
-- Orthogonal to `scopes`: even if scopes grant write/run, this flag denies
-- mutating methods (POST/PUT/PATCH/DELETE) and Run actions.
ALTER TABLE token ADD COLUMN read_only BOOLEAN NOT NULL DEFAULT false;
@@ -0,0 +1,3 @@
-- No-op: clearing a stray `auto_kind = 'lib'` value on failure/trigger/approval
-- scripts is not reversible (the original NULL/'lib' distinction is lost), and
-- restoring `'lib'` here would re-hide these scripts from their pickers.
@@ -0,0 +1,9 @@
-- Failure, Trigger, and Approval scripts are runnable entrypoints by
-- definition. A prior parser regression occasionally classified them as
-- `auto_kind = 'lib'`, which hid them from the flow error-handler /
-- trigger / approval pickers. Clear those stray values so existing
-- affected scripts re-appear without requiring a redeploy.
UPDATE script
SET auto_kind = NULL
WHERE auto_kind = 'lib'
AND kind IN ('failure', 'trigger', 'approval');
+24 -24
View File
@@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6263,7 +6263,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"proc-macro2",
"quote",
@@ -6275,7 +6275,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"convert_case",
"serde",
@@ -6284,7 +6284,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6296,7 +6296,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"serde_json",
@@ -6308,7 +6308,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"gosyn",
@@ -6320,7 +6320,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6332,7 +6332,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"serde_json",
@@ -6344,7 +6344,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"nu-parser",
@@ -6355,7 +6355,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6366,7 +6366,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6378,7 +6378,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6389,7 +6389,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -6411,7 +6411,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"serde_json",
@@ -6423,7 +6423,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6437,7 +6437,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"convert_case",
@@ -6454,7 +6454,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6467,7 +6467,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"serde",
@@ -6479,7 +6479,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -6497,7 +6497,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6513,7 +6513,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6529,7 +6529,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6561,7 +6561,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"serde",
@@ -6572,7 +6572,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.699.0"
version = "1.702.1"
dependencies = [
"anyhow",
"bitflags",
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.699.0"
version = "1.702.1"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
+81 -54
View File
@@ -6,6 +6,8 @@ use windmill_common::{
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
#[cfg(feature = "operator")]
pub const DEFAULT_MAX_CONNECTIONS_OPERATOR: u32 = 2;
pub async fn initial_connection() -> Result<sqlx::Pool<sqlx::Postgres>, error::Error> {
let connect_options = get_database_url().await?.connect_options().await?;
@@ -16,12 +18,35 @@ pub async fn initial_connection() -> Result<sqlx::Pool<sqlx::Postgres>, error::E
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))
}
/// Connect to the database for the Kubernetes operator process.
///
/// Long-running operator pods need IAM RDS / Entra ID token refresh just like the server,
/// otherwise new pool connections start failing once the initial token expires (~15 min).
#[cfg(feature = "operator")]
pub async fn operator_connection(
#[cfg(all(feature = "enterprise", feature = "private"))]
killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
let database_url = get_database_url().await?;
let pool = connect(
database_url.clone(),
DEFAULT_MAX_CONNECTIONS_OPERATOR,
false,
)
.await?;
#[cfg(all(feature = "enterprise", feature = "private"))]
spawn_token_refresh_task(pool.clone(), database_url, killpill_rx);
Ok(pool)
}
pub async fn connect_db(
server_mode: bool,
indexer_mode: bool,
worker_mode: bool,
num_workers: i32,
#[cfg(feature = "private")] mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
#[cfg(feature = "private")] killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
use anyhow::Context;
@@ -43,70 +68,72 @@ pub async fn connect_db(
let pool = connect(database_url.clone(), max_connections, worker_mode).await?;
#[cfg(all(feature = "enterprise", feature = "private"))]
{
let needs_token_refresh = matches!(
database_url,
DatabaseUrl::IamRds(_) | DatabaseUrl::EntraId(_)
);
let label = match &database_url {
DatabaseUrl::IamRds(_) => "IAM RDS",
DatabaseUrl::EntraId(_) => "Entra ID",
DatabaseUrl::Static(_) => "",
};
if needs_token_refresh {
let pool2 = pool.clone();
let database_url2 = database_url.clone();
tokio::spawn(async move {
loop {
tokio::select! {
_ = killpill_rx.recv() => {
break;
}
_ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
if !database_url2.needs_refresh().await {
continue;
}
let new_url = tokio::time::timeout(
std::time::Duration::from_secs(10),
get_database_url(),
)
.await;
match new_url {
Ok(Ok(new_url)) => {
match new_url.connect_options().await {
Ok(connect_options) => {
pool2.set_connect_options(connect_options);
tracing::info!("Refreshed {label} URL successfully");
}
Err(e) => {
tracing::error!(
"Error getting {label} connect options, retrying in 10s: {e}"
);
continue;
}
}
}
Ok(Err(e)) => {
tracing::error!(
"Error refreshing {label} URL, trying again in 10s: {e}"
);
continue;
spawn_token_refresh_task(pool.clone(), database_url, killpill_rx);
Ok(pool)
}
/// Spawn a background task that refreshes IAM RDS / Entra ID tokens before they expire
/// and updates the pool's connect options so new connections use the fresh token.
/// No-op for static (password-based) database URLs.
#[cfg(all(feature = "enterprise", feature = "private"))]
pub fn spawn_token_refresh_task(
pool: sqlx::Pool<sqlx::Postgres>,
database_url: DatabaseUrl,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
let label = match &database_url {
DatabaseUrl::IamRds(_) => "IAM RDS",
DatabaseUrl::EntraId(_) => "Entra ID",
DatabaseUrl::Static(_) => return,
};
tokio::spawn(async move {
loop {
tokio::select! {
_ = killpill_rx.recv() => {
break;
}
_ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
if !database_url.needs_refresh().await {
continue;
}
let new_url = tokio::time::timeout(
std::time::Duration::from_secs(10),
get_database_url(),
)
.await;
match new_url {
Ok(Ok(new_url)) => {
match new_url.connect_options().await {
Ok(connect_options) => {
pool.set_connect_options(connect_options);
tracing::info!("Refreshed {label} URL successfully");
}
Err(e) => {
tracing::error!(
"Timeout after 10s refreshing {label} URL, trying again in 10s: {e}"
"Error getting {label} connect options, retrying in 10s: {e}"
);
continue;
}
}
}
Ok(Err(e)) => {
tracing::error!(
"Error refreshing {label} URL, trying again in 10s: {e}"
);
continue;
}
Err(e) => {
tracing::error!(
"Timeout after 10s refreshing {label} URL, trying again in 10s: {e}"
);
continue;
}
}
}
});
}
}
}
Ok(pool)
});
}
pub async fn connect(
+18 -1
View File
@@ -670,7 +670,24 @@ async fn windmill_main() -> anyhow::Result<()> {
tracing_subscriber::fmt::init();
tracing::info!("Starting Windmill Kubernetes operator...");
tracing::info!("Connecting to database...");
let db = crate::db_connect::initial_connection().await?;
#[cfg(all(feature = "enterprise", feature = "private"))]
let (operator_killpill_tx, operator_killpill_rx) =
tokio::sync::broadcast::channel::<()>(2);
let db = crate::db_connect::operator_connection(
#[cfg(all(feature = "enterprise", feature = "private"))]
operator_killpill_rx,
)
.await?;
#[cfg(all(feature = "enterprise", feature = "private"))]
tokio::spawn(async move {
if let Ok(()) = tokio::signal::ctrl_c().await {
let _ = operator_killpill_tx.send(());
}
});
tracing::info!("Database connected. Starting ConfigMap watcher...");
windmill_operator::run(db).await?;
return Ok(());
+4 -2
View File
@@ -886,11 +886,13 @@ pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) {
let mut current = OTEL_TRACING_PROXY_SETTINGS.write().await;
if current.enabled != new_settings.enabled
|| current.enabled_languages != new_settings.enabled_languages
|| current.no_proxy_hosts != new_settings.no_proxy_hosts
{
tracing::info!(
"OTEL tracing proxy settings changed: enabled={}, languages={:?}",
"OTEL tracing proxy settings changed: enabled={}, languages={:?}, no_proxy_hosts={:?}",
new_settings.enabled,
new_settings.enabled_languages
new_settings.enabled_languages,
new_settings.no_proxy_hosts,
);
*current = new_settings;
}
+57
View File
@@ -577,6 +577,63 @@ export function main() {
Ok(())
}
/// Regression test: a `//nobundling` script that pulls a package whose CJS
/// internals do bare-specifier `require()` of a sibling dependency.
///
/// Before the `--preserve-symlinks` fix, Bun 1.2/1.3+ would follow the
/// directory symlink in `node_modules/@langchain/core` to its global cache
/// entry, walk parent dirs from the cache realpath, and fail to find
/// `node_modules/zod` — producing:
/// ENOENT while resolving package 'zod/v3' from
/// '.../cache_nomount/bun/@langchain/core@<ver>@@@1/dist/runnables/base.js'
///
/// The fix passes `--preserve-symlinks` so Bun resolves from the
/// symlink path under `<job_dir>/node_modules/`, where `zod` is a sibling.
#[sqlx::test(fixtures("base"))]
async fn test_bun_nobundling_transitive_require(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let content = r#"//nobundling
import { ChatPromptTemplate } from "@langchain/core/prompts";
export async function main() {
const tpl = ChatPromptTemplate.fromMessages([
["system", "you are a {role}"],
["human", "{input}"],
]);
const out = await tpl.formatMessages({ role: "tester", input: "ping" });
return out.length;
}
"#
.to_owned();
let job = JobPayload::Code(RawCode {
hash: None,
content,
path: None,
language: ScriptLang::Bun,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
modules: None,
tag: None,
});
let result = run_job_in_new_worker_until_complete(&db, false, job, port)
.await
.json_result()
.unwrap();
assert_eq!(result, serde_json::json!(2));
Ok(())
}
// ============================================================================
// Native Mode Tests (requires deno_core feature)
// ============================================================================
+106
View File
@@ -760,6 +760,112 @@ def main():
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base"))]
async fn test_python_result_preserves_infinity_in_string(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let content = r#"
def main():
return {
"plain": "Infinity",
"embedded": "value=-Infinity end",
"nan_word": "this is NaN inside text",
"nested": [{"k": "Infinity"}],
}
"#
.to_owned();
let job = JobPayload::Code(RawCode {
hash: None,
content,
path: None,
language: ScriptLang::Python3,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
modules: None,
tag: None,
});
let result = run_job_in_new_worker_until_complete(&db, false, job, port)
.await
.json_result()
.unwrap();
assert_eq!(
result,
serde_json::json!({
"plain": "Infinity",
"embedded": "value=-Infinity end",
"nan_word": "this is NaN inside text",
"nested": [{"k": "Infinity"}],
})
);
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base"))]
async fn test_python_result_non_finite_floats_become_null(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let content = r#"
def main():
return {
"inf": float("inf"),
"neg_inf": float("-inf"),
"nan": float("nan"),
"finite": 1.5,
"nested": [float("inf"), {"x": float("nan")}],
}
"#
.to_owned();
let job = JobPayload::Code(RawCode {
hash: None,
content,
path: None,
language: ScriptLang::Python3,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
modules: None,
tag: None,
});
let result = run_job_in_new_worker_until_complete(&db, false, job, port)
.await
.json_result()
.unwrap();
assert_eq!(
result,
serde_json::json!({
"inf": null,
"neg_inf": null,
"nan": null,
"finite": 1.5,
"nested": [null, {"x": null}],
})
);
Ok(())
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base"))]
async fn test_python_global_site_packages(db: Pool<Postgres>) -> anyhow::Result<()> {
+122
View File
@@ -0,0 +1,122 @@
use std::collections::HashMap;
use sqlx::{Pool, Postgres};
use windmill_api_client::types::{NewScript, ScriptLang};
use windmill_test_utils::init_client;
fn quick_ns(content: &str, path: &str, kind: Option<&str>) -> NewScript {
NewScript {
content: content.into(),
language: ScriptLang::Bun,
lock: None,
parent_hash: None,
path: path.into(),
concurrent_limit: None,
concurrency_time_window_s: None,
cache_ttl: None,
dedicated_worker: None,
description: "".to_string(),
draft_only: None,
envs: vec![],
is_template: None,
kind: kind.map(|s| s.to_string()),
summary: "".to_string(),
tag: None,
schema: HashMap::new(),
ws_error_handler_muted: Some(false),
priority: None,
delete_after_secs: None,
timeout: None,
restart_unless_cancelled: None,
deployment_message: None,
concurrency_key: None,
visible_to_runner_only: None,
auto_kind: None,
codebase: None,
has_preprocessor: None,
on_behalf_of_email: None,
assets: vec![],
modules: None,
}
}
/// Regression: a `failure`-kind script must never be marked `auto_kind = 'lib'`
/// even if the parser fails to detect a `main` function, because the flow
/// error-handler picker filters out lib scripts and would otherwise hide it.
#[sqlx::test(fixtures("base"))]
async fn failure_kind_script_without_main_is_not_marked_lib(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
let (client, _port, _s) = init_client(db.clone()).await;
// Content with no `main` — TS parser would normally set auto_kind = 'lib'.
client
.create_script(
"test-workspace",
&quick_ns(
"export function notMain() { return 42 }",
"u/test-user/failure_no_main",
Some("failure"),
),
)
.await
.unwrap();
let auto_kind: Option<String> = sqlx::query_scalar(
"SELECT auto_kind FROM script \
WHERE workspace_id = $1 AND path = $2",
)
.bind("test-workspace")
.bind("u/test-user/failure_no_main")
.fetch_one(&db)
.await?;
assert_ne!(
auto_kind.as_deref(),
Some("lib"),
"failure-kind script must not be marked as 'lib' auto_kind, got {:?}",
auto_kind
);
Ok(())
}
/// Sibling: a normal `script` kind WITHOUT main should still be marked `lib`
/// (so it stays hidden from the regular script picker). Guards against an
/// over-broad sanitizer accidentally clearing the value for plain scripts.
#[sqlx::test(fixtures("base"))]
async fn regular_script_without_main_is_still_marked_lib(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
let (client, _port, _s) = init_client(db.clone()).await;
client
.create_script(
"test-workspace",
&quick_ns(
"export function notMain() { return 42 }",
"u/test-user/script_no_main",
Some("script"),
),
)
.await
.unwrap();
let auto_kind: Option<String> = sqlx::query_scalar(
"SELECT auto_kind FROM script \
WHERE workspace_id = $1 AND path = $2",
)
.bind("test-workspace")
.bind("u/test-user/script_no_main")
.fetch_one(&db)
.await?;
assert_eq!(
auto_kind.as_deref(),
Some("lib"),
"regular script without main should be marked 'lib', got {:?}",
auto_kind
);
Ok(())
}
@@ -174,6 +174,7 @@ fn make_authed() -> windmill_api_auth::ApiAuthed {
scopes: None,
username_override: None,
token_prefix: None,
read_only: false,
}
}
+34 -2
View File
@@ -326,8 +326,10 @@ pub fn json_to_document(value: serde_json::Value) -> aws_smithy_types::Document
}
Value::Array(arr) => Document::Array(arr.into_iter().map(json_to_document).collect()),
Value::Number(num) => {
if let Some(i) = num.as_i64() {
Document::Number(aws_smithy_types::Number::PosInt(i as u64))
if let Some(u) = num.as_u64() {
Document::Number(aws_smithy_types::Number::PosInt(u))
} else if let Some(i) = num.as_i64() {
Document::Number(aws_smithy_types::Number::NegInt(i))
} else if let Some(f) = num.as_f64() {
Document::Number(aws_smithy_types::Number::Float(f))
} else {
@@ -844,6 +846,36 @@ mod tests {
}
}
#[test]
fn json_to_document_preserves_negative_integers() {
let value = serde_json::json!(-1);
let doc = json_to_document(value);
assert!(matches!(
doc,
aws_smithy_types::Document::Number(aws_smithy_types::Number::NegInt(-1))
));
}
#[test]
fn json_to_document_handles_large_u64_above_i64_max() {
let value = serde_json::json!(u64::MAX);
let doc = json_to_document(value);
assert!(matches!(
doc,
aws_smithy_types::Document::Number(aws_smithy_types::Number::PosInt(u)) if u == u64::MAX
));
}
#[test]
fn json_to_document_handles_positive_integers() {
let value = serde_json::json!(42);
let doc = json_to_document(value);
assert!(matches!(
doc,
aws_smithy_types::Document::Number(aws_smithy_types::Number::PosInt(42))
));
}
#[test]
fn openai_messages_to_bedrock_adds_cache_points_when_enabled() {
let messages = vec![
+1
View File
@@ -5,6 +5,7 @@ pub mod ai_google;
pub mod ai_providers;
pub mod ai_types;
pub mod image_handler;
pub mod providers;
pub mod query_builder;
pub mod sse;
pub mod types;
@@ -1,7 +1,4 @@
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use windmill_ai::{
use crate::{
ai_google::parse_data_url,
ai_providers::AIProvider,
image_handler::prepare_messages_for_api,
@@ -10,6 +7,9 @@ use windmill_ai::{
types::*,
utils::{extract_text_content, should_use_structured_output_tool},
};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use windmill_common::{client::AuthedClient, error::Error};
/// Anthropic API version for standard API
@@ -1,4 +1,4 @@
//! AWS Bedrock provider for the AI agent.
//! AWS Bedrock provider for AI requests.
//!
//! Uses shared SDK code from windmill_ai::ai_bedrock for:
//! - BedrockClient (SDK wrapper with auth)
@@ -6,16 +6,16 @@
//! - Stream event parsing
//! - Helper utilities
use std::collections::HashMap;
use windmill_ai::{
use crate::{
image_handler::prepare_messages_for_api,
query_builder::{ParsedResponse, StreamEventSink},
types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef},
};
use std::collections::HashMap;
use windmill_common::{client::AuthedClient, error::Error};
// Import shared Bedrock helpers for worker-specific orchestration.
use windmill_ai::ai_bedrock::{
// Import shared Bedrock helpers for provider orchestration.
use crate::ai_bedrock::{
bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop,
bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta,
bedrock_stream_event_to_tool_start, build_tool_config, create_inference_config,
@@ -24,7 +24,7 @@ use windmill_ai::ai_bedrock::{
};
// ============================================================================
// Query Builder (Worker-specific orchestration)
// Query Builder
// ============================================================================
#[derive(Default)]
@@ -1,5 +1,4 @@
use async_trait::async_trait;
use windmill_ai::{
use crate::{
ai_google::{
openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig,
GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart,
@@ -10,6 +9,7 @@ use windmill_ai::{
sse::{GeminiSSEParser, SSEParser},
types::*,
};
use async_trait::async_trait;
use windmill_common::{client::AuthedClient, error::Error};
// ============================================================================
+31
View File
@@ -0,0 +1,31 @@
pub mod anthropic;
#[cfg(feature = "bedrock")]
pub mod bedrock;
pub mod google_ai;
pub mod openai;
pub mod openrouter;
pub mod other;
use crate::{ai_providers::AIProvider, query_builder::QueryBuilder, types::ProviderWithResource};
use self::{
anthropic::AnthropicQueryBuilder, google_ai::GoogleAIQueryBuilder, openai::OpenAIQueryBuilder,
openrouter::OpenRouterQueryBuilder, other::OtherQueryBuilder,
};
/// Factory function to create the appropriate query builder for a provider.
pub fn create_query_builder(provider: &ProviderWithResource) -> Box<dyn QueryBuilder> {
match provider.kind {
AIProvider::GoogleAI => {
Box::new(GoogleAIQueryBuilder::new(provider.get_platform().clone()))
}
AIProvider::OpenAI => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())),
AIProvider::Anthropic => Box::new(AnthropicQueryBuilder::new(
provider.kind.clone(),
provider.get_platform().clone(),
provider.get_enable_1m_context(),
)),
AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()),
_ => Box::new(OtherQueryBuilder::new(provider.kind.clone())),
}
}
@@ -1,7 +1,4 @@
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use windmill_ai::{
use crate::{
ai_providers::AIProvider,
ai_types::OpenAIToolCall,
image_handler::{prepare_messages_for_api, s3_object_to_content_part},
@@ -10,6 +7,9 @@ use windmill_ai::{
types::*,
utils::extract_text_content,
};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use windmill_common::{client::AuthedClient, error::Error};
// Responses API structures
@@ -1,15 +1,15 @@
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json;
use windmill_ai::{
use crate::{
ai_providers::AIProvider,
image_handler::prepare_messages_for_api,
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink},
types::*,
};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json;
use windmill_common::{client::AuthedClient, error::Error};
use crate::ai::providers::other::OtherQueryBuilder;
use crate::providers::other::OtherQueryBuilder;
// OpenRouter-specific types
#[derive(Serialize)]
@@ -1,7 +1,4 @@
use async_trait::async_trait;
use serde::Serialize;
use serde_json;
use windmill_ai::{
use crate::{
ai_providers::AIProvider,
image_handler::prepare_messages_for_api,
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink},
@@ -9,6 +6,9 @@ use windmill_ai::{
types::*,
utils::should_use_structured_output_tool,
};
use async_trait::async_trait;
use serde::Serialize;
use serde_json;
use windmill_common::{client::AuthedClient, error::Error};
#[derive(Serialize, Debug, Clone)]
+47 -7
View File
@@ -194,6 +194,7 @@ impl AuthCache {
scopes: None,
username_override,
token_prefix: claims.audit_span,
read_only: false,
};
let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok());
AUTH_CACHE.insert(
@@ -221,11 +222,20 @@ impl AuthCache {
token_hash = $1
AND (expiration > NOW() OR expiration IS NULL)
AND (workspace_id IS NULL OR workspace_id = $2)
RETURNING owner, email, super_admin, scopes, label",
RETURNING owner, email, super_admin, scopes, label, read_only",
t_hash,
w_id.as_ref(),
)
.map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label))
.map(|x| {
(
x.owner,
x.email,
x.super_admin,
x.scopes,
x.label,
x.read_only,
)
})
.fetch_optional(&self.db)
.await
.ok()
@@ -234,7 +244,9 @@ impl AuthCache {
if let Some(user) = user_o {
let authed_o = {
match user {
(Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
(Some(owner), Some(email), super_admin, _, label, read_only)
if w_id.is_some() =>
{
let username_override = username_override_from_label(label);
if let Some((prefix, name)) = owner.split_once('/') {
if prefix == "u" {
@@ -280,6 +292,7 @@ impl AuthCache {
scopes: None,
username_override,
token_prefix: Some(safe_token_prefix(token)),
read_only,
})
} else {
let groups = vec![name.to_string()];
@@ -305,6 +318,7 @@ impl AuthCache {
scopes: None,
username_override,
token_prefix: Some(safe_token_prefix(token)),
read_only,
})
}
} else {
@@ -320,10 +334,11 @@ impl AuthCache {
scopes: None,
username_override,
token_prefix: Some(safe_token_prefix(token)),
read_only,
})
}
}
(_, Some(email), super_admin, scopes, label) => {
(_, Some(email), super_admin, scopes, label, read_only) => {
let username_override = username_override_from_label(label);
if w_id.is_some() {
let row_o = sqlx::query!(
@@ -368,6 +383,7 @@ impl AuthCache {
scopes,
username_override,
token_prefix: Some(safe_token_prefix(token)),
read_only,
})
}
None if super_admin => Some(ApiAuthed {
@@ -380,6 +396,7 @@ impl AuthCache {
scopes,
username_override,
token_prefix: Some(safe_token_prefix(token)),
read_only,
}),
None => None,
}
@@ -394,6 +411,7 @@ impl AuthCache {
scopes,
username_override,
token_prefix: Some(safe_token_prefix(token)),
read_only,
})
}
}
@@ -428,6 +446,7 @@ impl AuthCache {
scopes: None,
username_override: None,
token_prefix: Some(safe_token_prefix(token)),
read_only: false,
};
Some(OptJobAuthed { authed, job_id: None })
} else {
@@ -630,6 +649,7 @@ pub async fn resolve_opt_job_authed(
scopes: None,
username_override: None,
token_prefix: None,
read_only: false,
};
return Ok((OptJobAuthed { authed, job_id: None }, parts));
}
@@ -667,12 +687,11 @@ pub async fn resolve_opt_job_authed(
cache.get_opt_job_authed(workspace_id.clone(), &token).await
{
let authed = &mut opt_job_authed.authed;
let path = original_uri.path();
let method = parts.method.as_str();
if authed.scopes.is_some() {
transform_old_scope_to_new_scope(authed.scopes.as_mut());
let path = original_uri.path();
let method = parts.method.as_str();
if let Err(err) = crate::scopes::check_scopes_for_route(
authed.scopes.as_deref(),
path,
@@ -681,6 +700,27 @@ pub async fn resolve_opt_job_authed(
return Err((err, parts));
}
}
if authed.read_only {
// MCP transport runs over POST (streamable HTTP / SSE handshake),
// so the middleware can't safely reject mutating methods on it —
// the MCP runner itself filters out write tools and rejects
// mutating tool calls for read-only tokens. Narrow to the actual
// transport endpoints: anything else under `/api/mcp/*` (OAuth
// approve, token exchange, client registration) must still go
// through the read-only check, otherwise a read-only token
// could approve an OAuth flow that mints a new non-read-only
// token.
let is_mcp_transport = path == "/api/mcp/gateway"
|| (path.starts_with("/api/mcp/w/")
&& (path.ends_with("/mcp")
|| path.ends_with("/sse")
|| path.ends_with("/list_tools")));
if !is_mcp_transport {
if let Err(err) = crate::scopes::check_read_only_for_route(path, method) {
return Err((err, parts));
}
}
}
parts.extensions.insert(authed.clone());
Span::current().record("username", &authed.username.as_str());
+14 -3
View File
@@ -55,6 +55,7 @@ pub struct ApiAuthed {
pub scopes: Option<Vec<String>>,
pub username_override: Option<String>,
pub token_prefix: Option<String>,
pub read_only: bool,
}
impl ApiAuthed {
@@ -103,6 +104,7 @@ impl From<Authed> for ApiAuthed {
scopes: value.scopes,
username_override: None,
token_prefix: value.token_prefix,
read_only: false,
}
}
}
@@ -183,6 +185,10 @@ impl windmill_mcp::server::McpAuth for ApiAuthed {
fn scopes(&self) -> Option<&[String]> {
self.scopes.as_deref()
}
fn read_only(&self) -> bool {
self.read_only
}
}
// ------------ Utility functions ------------
@@ -478,6 +484,7 @@ pub async fn fetch_api_authed_from_permissioned_as(
scopes: authed.scopes,
username_override: None,
token_prefix: authed.token_prefix,
read_only: false,
};
API_AUTHED_CACHE.insert(
@@ -506,6 +513,8 @@ pub struct NewToken {
pub impersonate_email: Option<String>,
pub scopes: Option<Vec<String>>,
pub workspace_id: Option<String>,
#[serde(default)]
pub read_only: Option<bool>,
}
impl NewToken {
@@ -515,8 +524,9 @@ impl NewToken {
impersonate_email: Option<String>,
scopes: Option<Vec<String>>,
workspace_id: Option<String>,
read_only: Option<bool>,
) -> Self {
Self { label, expiration, impersonate_email, scopes, workspace_id }
Self { label, expiration, impersonate_email, scopes, workspace_id, read_only }
}
}
@@ -564,8 +574,8 @@ pub async fn create_token_internal(
}
let rows = sqlx::query!(
"INSERT INTO token
(token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id)
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9
(token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id, read_only)
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10
WHERE $9::varchar IS NULL OR NOT EXISTS(
SELECT 1 FROM workspace WHERE id = $9 AND deleted = true
)",
@@ -578,6 +588,7 @@ pub async fn create_token_internal(
is_super_admin,
token_config.scopes.as_ref().map(|x| x.as_slice()),
token_config.workspace_id,
token_config.read_only.unwrap_or(false),
)
.execute(&mut *tx)
.await?;
+41
View File
@@ -686,6 +686,19 @@ fn scope_grants_access(
Ok(true)
}
/// Enforces a token's `read_only` flag: only methods classified as `Read`
/// (GET/HEAD/OPTIONS) are allowed. Run actions and mutating methods are
/// rejected. Independent of `scopes`.
pub fn check_read_only_for_route(route_path: &str, http_method: &str) -> Result<()> {
if map_http_method_to_action(http_method, route_path) == ScopeAction::Read {
Ok(())
} else {
Err(Error::PermissionDenied(
"Token is read-only. Mutating endpoints are not allowed.".to_string(),
))
}
}
/// Helper function to check if scopes allow access to a route
pub fn check_scopes_for_route(
token_scopes: Option<&[String]>,
@@ -778,6 +791,34 @@ mod tests {
assert_eq!(route_suffix, Some("flow_conversations/list".to_string()));
}
#[test]
fn test_check_read_only_for_route() {
// Plain GETs pass.
assert!(check_read_only_for_route("/api/w/x/scripts/list", "GET").is_ok());
assert!(check_read_only_for_route("/api/w/x/scripts/get/foo", "HEAD").is_ok());
assert!(check_read_only_for_route("/api/w/x/anything", "OPTIONS").is_ok());
// Mutating methods are rejected.
assert!(check_read_only_for_route("/api/w/x/scripts/create", "POST").is_err());
assert!(check_read_only_for_route("/api/w/x/scripts/update", "PUT").is_err());
assert!(check_read_only_for_route("/api/w/x/scripts/delete", "DELETE").is_err());
assert!(check_read_only_for_route("/api/w/x/scripts/patch", "PATCH").is_err());
// Run paths are rejected even on GET (map_http_method_to_action elevates
// them to Run via RUN_PATH_ACTIONS).
assert!(check_read_only_for_route("/api/w/x/jobs/run/p/f/foo", "GET").is_err());
assert!(check_read_only_for_route("/api/w/x/jobs/run/p/f/foo", "POST").is_err());
// OAuth/registration endpoints under /api/mcp/* must NOT be exempted by
// the auth middleware — they go through this check on the gateway side
// because they can mint non-read-only tokens. The middleware decides
// which paths to exempt; this helper is method-only, so we just assert
// that mutating methods still fail.
assert!(
check_read_only_for_route("/api/mcp/gateway/oauth/server/approve", "POST").is_err()
);
}
#[test]
fn test_specific_scope_access() {
let scopes = vec!["jobs:read".to_string()];
+219 -84
View File
@@ -12,6 +12,9 @@ use axum::{
Json, Router,
};
use windmill_api_auth::require_owner_of_path;
use windmill_audit::audit_oss::audit_log;
use windmill_audit::ActionKind;
use windmill_common::scripts::ScriptHash;
use windmill_common::DB;
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
@@ -24,6 +27,31 @@ use windmill_common::{
utils::{not_found_if_none, StripPath},
};
/// Map a granular-ACL kind segment to the audit-log action prefix used by the
/// per-kind CRUD endpoints (e.g. `flows.update`, `scripts.update`). The
/// resulting action is suffixed with `.grant_acl` / `.revoke_acl` so the
/// audit log keeps a per-resource record of every `/acls/*` mutation —
/// folder/group already log via their dedicated permission-history tables.
fn audit_action_prefix_for_acl_kind(kind: &str) -> Option<&'static str> {
match kind {
"script" => Some("scripts"),
"flow" => Some("flows"),
"app" => Some("apps"),
// Distinct prefix so dashboards aggregating on `action` can separate
// raw_app ACL mutations from regular app ones without parsing the
// `kind` parameters field. (The granular_acls SQL routes raw_app
// writes to the same `app` table; audit log identity is separate.)
"raw_app" => Some("raw_apps"),
"resource" => Some("resources"),
"variable" => Some("variables"),
"schedule" => Some("schedules"),
"http_trigger" | "websocket_trigger" | "kafka_trigger" | "nats_trigger"
| "postgres_trigger" | "mqtt_trigger" | "gcp_trigger" | "azure_trigger" | "sqs_trigger"
| "email_trigger" => Some("triggers"),
_ => None,
}
}
const KINDS: [&str; 20] = [
"script",
"group_",
@@ -131,9 +159,15 @@ async fn add_granular_acl(
}
}
// v2 raw apps are stored in the `app` table (with `app_version.raw_app = true`
// distinguishing them from regular apps); the legacy `raw_app` table no longer
// backs the workspace export, so granting/revoking on `raw_app` must hit `app`
// for the change to be visible. Git-sync dispatch still uses
// DeployedObject::RawApp so the worker writes back `<path>.raw_app.json`.
let table = if kind == "raw_app" { "app" } else { kind };
// SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function.
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
"UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2), \
"UPDATE {table} SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2), \
true) WHERE {identifier} = $3 AND workspace_id = $4 RETURNING extra_perms"
))
.bind(vec![owner.clone()])
@@ -175,6 +209,34 @@ async fn add_granular_acl(
Some(&owner),
)
.await?;
} else if let Some(prefix) = audit_action_prefix_for_acl_kind(kind) {
// Mirror the folder/group permission-history coverage for every other
// ACLable kind. Folder/group already wrote a dedicated history row
// above; everything else (script/flow/app/raw_app/resource/...) lands
// in the general audit_log table here.
let access = if write.unwrap_or(false) {
"write"
} else {
"read"
};
let action = format!("{}.grant_acl", prefix);
audit_log(
&mut *tx,
&authed,
action.as_str(),
ActionKind::Update,
&w_id,
Some(path),
Some(
[
("kind", kind),
("owner", owner.as_str()),
("access", access),
]
.into(),
),
)
.await?;
}
tx.commit().await?;
@@ -193,46 +255,67 @@ async fn add_granular_acl(
)
.await?
}
// "app" => {
// handle_deployment_metadata(
// &authed.email,
// &authed.username,
// &db,
// &w_id,
// DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
// Some(format!("App '{}' changed permissions", path)),
// // true,
// )
// .await?
// }
// "script" => {
// handle_deployment_metadata(
// &authed.email,
// &authed.username,
// &db,
// &w_id,
// DeployedObject::Script {
// path: path.to_string(),
// parent_path: None,
// hash: ScriptHash(0),
// },
// Some(format!("Script '{}' changed permissions", path)),
// // true,
// )
// .await?
// }
// "flow" => {
// handle_deployment_metadata(
// &authed.email,
// &authed.username,
// &db,
// &w_id,
// DeployedObject::Flow { path: path.to_string(), parent_path: None },
// Some(format!("Flow '{}' changed permissions", path)),
// // true,
// )
// .await?
// }
"app" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
Some(format!("App '{}' changed permissions", path)),
true,
None,
)
.await?
}
"raw_app" => {
// RawApp deliberately uses its own DeployedObject variant: the
// git-sync worker reads `path_type` ("app" vs "raw_app") to decide
// whether to write `<path>.app.json` or `<path>.raw_app.json`.
// Collapsing this into `App` would dispatch raw_app perm changes
// against the wrong file in the repo.
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::RawApp { path: path.to_string(), parent_path: None, version: 0 },
Some(format!("Raw App '{}' changed permissions", path)),
true,
None,
)
.await?
}
"script" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Script {
path: path.to_string(),
parent_path: None,
hash: ScriptHash(0),
},
Some(format!("Script '{}' changed permissions", path)),
true,
None,
)
.await?
}
"flow" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Flow { path: path.to_string(), parent_path: None, version: 0 },
Some(format!("Flow '{}' changed permissions", path)),
true,
None,
)
.await?
}
_ => (),
}
@@ -295,13 +378,17 @@ async fn remove_granular_acl(
require_owner_of_path(&authed, path)?;
}
// See add_granular_acl: kind="raw_app" must hit the `app` table because v2
// raw apps live there and the legacy `raw_app` table no longer backs the
// workspace export.
let table = if kind == "raw_app" { "app" } else { kind };
// SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function.
let obj_o = sqlx::query_scalar::<_, bool>(&format!(
"WITH old AS (
SELECT extra_perms->$1 as old_write FROM {kind}
SELECT extra_perms->$1 as old_write FROM {table}
WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1
)
UPDATE {kind} SET extra_perms = extra_perms - $1
UPDATE {table} SET extra_perms = extra_perms - $1
WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1
RETURNING (SELECT old_write FROM old)::bool"
))
@@ -335,6 +422,28 @@ async fn remove_granular_acl(
Some(&owner),
)
.await?;
} else if let Some(prefix) = audit_action_prefix_for_acl_kind(kind) {
// Mirror the add path: standard audit_log row for every kind that
// doesn't have a dedicated permission-history table.
let access = if write { "write" } else { "read" };
let action = format!("{}.revoke_acl", prefix);
audit_log(
&mut *tx,
&authed,
action.as_str(),
ActionKind::Update,
&w_id,
Some(path),
Some(
[
("kind", kind),
("owner", owner.as_str()),
("access", access),
]
.into(),
),
)
.await?;
}
tx.commit().await?;
@@ -353,46 +462,68 @@ async fn remove_granular_acl(
)
.await?
}
// "app" => {
// handle_deployment_metadata(
// &authed.email,
// &authed.username,
// &db,
// &w_id,
// DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
// Some(format!("App '{}' changed permissions", path)),
// // true,
// )
// .await?
// }
// "script" => {
// handle_deployment_metadata(
// &authed.email,
// &authed.username,
// &db,
// &w_id,
// DeployedObject::Script {
// path: path.to_string(),
// parent_path: None,
// hash: ScriptHash(0),
// },
// Some(format!("Script '{}' changed permissions", path)),
// // true,
// )
// .await?
// }
// "flow" => {
// handle_deployment_metadata(
// &authed.email,
// &authed.username,
// &db,
// &w_id,
// DeployedObject::Flow { path: path.to_string(), parent_path: None },
// Some(format!("Flow '{}' changed permissions", path)),
// // true,
// )
// .await?
// }
"app" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
Some(format!("App '{}' changed permissions", path)),
true,
None,
)
.await?
}
"raw_app" => {
// See add_granular_acl: raw_app must use its own DeployedObject
// variant so git-sync writes `<path>.raw_app.json`.
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::RawApp {
path: path.to_string(),
parent_path: None,
version: 0,
},
Some(format!("Raw App '{}' changed permissions", path)),
true,
None,
)
.await?
}
"script" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Script {
path: path.to_string(),
parent_path: None,
hash: ScriptHash(0),
},
Some(format!("Script '{}' changed permissions", path)),
true,
None,
)
.await?
}
"flow" => {
handle_deployment_metadata(
&authed.email,
&authed.username,
&db,
&w_id,
DeployedObject::Flow { path: path.to_string(), parent_path: None, version: 0 },
Some(format!("Flow '{}' changed permissions", path)),
true,
None,
)
.await?
}
_ => (),
}
}
@@ -421,9 +552,13 @@ async fn get_granular_acls(
} else {
"path"
};
// See add_granular_acl: raw_app rows live in the `app` table now, so the
// read path must also target `app` — otherwise GET would return stale or
// 404 state while POST /acls/add and /acls/remove write to `app`.
let table = if kind == "raw_app" { "app" } else { kind };
// SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function.
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
"SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2"
"SELECT extra_perms from {table} WHERE {identifier} = $1 AND workspace_id = $2"
))
.bind(path)
.bind(w_id)
@@ -51,6 +51,7 @@ fn test_authed() -> ApiAuthed {
scopes: None,
username_override: None,
token_prefix: None,
read_only: false,
}
}
@@ -573,8 +573,10 @@ async fn test_promotion_individual_branch_debounces_per_path(
// Wait for both deployment callbacks to resolve a debounce key. The
// alpha key is polled first as a warm-up, then we also wait for beta
// so the assertions don't race the second spawned callback.
let expected_alpha = "git_sync:script:f/target/alpha";
let expected_beta = "git_sync:script:f/target/beta";
// Keys are namespaced by the repo's resource path so multiple promotion
// repos don't collide on the same key.
let expected_alpha = "git_sync:$res:u/test-user/test_git_repo:script:f/target/alpha";
let expected_beta = "git_sync:$res:u/test-user/test_git_repo:script:f/target/beta";
let _ = wait_for_debounce_key(&db, expected_alpha, Duration::from_secs(5)).await?;
let keys = wait_for_debounce_key(&db, expected_beta, Duration::from_secs(5)).await?;
assert!(
@@ -589,6 +591,162 @@ async fn test_promotion_individual_branch_debounces_per_path(
Ok(())
}
/// Create a second git repository resource for multi-repo tests.
#[allow(dead_code)]
async fn create_second_git_repo_resource(db: &Pool<Postgres>) -> anyhow::Result<()> {
sqlx::query(
r#"
INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by)
VALUES ('test-workspace', 'u/test-user/test_git_repo_2', $1::jsonb, 'git_repository', '{}'::jsonb, 'test-user')
ON CONFLICT (workspace_id, path) DO NOTHING
"#,
)
.bind(json!({
"url": "https://github.com/test/test2.git",
"branch": "main",
"token": "test-token-2"
}))
.execute(db)
.await?;
Ok(())
}
/// Configure git sync with TWO promotion-mode repositories pointing at distinct
/// git repo resources. Both repos use the same sync script and the same item
/// filters — they only differ in the repo they target.
#[allow(dead_code)]
async fn setup_two_promotion_repos_config(
db: &Pool<Postgres>,
sync_script_path: &str,
group_by_folder: bool,
) -> anyhow::Result<()> {
let git_sync_config = json!({
"include_type": ["script"],
"include_path": ["**"],
"repositories": [
{
"script_path": sync_script_path,
"git_repo_resource_path": "$res:u/test-user/test_git_repo",
"use_individual_branch": true,
"group_by_folder": group_by_folder
},
{
"script_path": sync_script_path,
"git_repo_resource_path": "$res:u/test-user/test_git_repo_2",
"use_individual_branch": true,
"group_by_folder": group_by_folder
}
]
});
sqlx::query!(
"UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
git_sync_config,
"test-workspace"
)
.execute(db)
.await?;
Ok(())
}
/// Regression test for: when two promotion-mode repos are configured with the
/// same `use_individual_branch=true` settings (i.e. a primary and a secondary
/// promotion repo), deploying a single script must enqueue ONE deployment
/// callback per repo. Both callbacks must remain in the queue — neither may
/// be debounced into oblivion by the other.
///
/// The bug this guards against: the debounce key for promotion mode was
/// derived only from (path_type, path) and omitted any per-repo identifier,
/// so the second repo's push hit ON CONFLICT in `upsert_debounce_key` and
/// `complete_debounced_job` flagged the first repo's job as `status='skipped'`
/// — silently dropping one of the two pushes.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_two_promotion_repos_both_enqueue_callback(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
create_folder(&db, "28103").await?;
create_folder(&db, "target").await?;
create_git_repo_resource(&db).await?;
create_second_git_repo_resource(&db).await?;
let sync_script_path = "f/28103/test_sync_two_promotion_repos";
create_sync_script(&db, sync_script_path).await?;
setup_two_promotion_repos_config(&db, sync_script_path, false).await?;
let (client, _port, _server) = init_client(db.clone()).await;
// Deploy a single script — handle_deployment_metadata should iterate
// both repos and create one callback job per repo.
create_test_script(&client, "f/target/alpha").await?;
// Both callbacks should reach the queue. With the bug, only one survives
// (the other is moved to v2_job_completed with status='skipped').
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
let mut last_jobs: Vec<DeploymentCallbackJob> = vec![];
loop {
last_jobs =
get_deployment_callback_jobs(&db, sync_script_path, Duration::from_millis(200)).await?;
if last_jobs.len() >= 2 {
break;
}
if tokio::time::Instant::now() >= deadline {
break;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
// Inspect what landed in v2_job_completed so failure messages explain why.
let skipped: Vec<(uuid::Uuid, String)> = sqlx::query_as(
r#"
SELECT c.id, c.status::text
FROM v2_job_completed c
JOIN v2_job j ON j.id = c.id
WHERE j.runnable_path = $1 AND j.kind = 'deploymentcallback'
"#,
)
.bind(sync_script_path)
.fetch_all(&db)
.await?;
assert_eq!(
last_jobs.len(),
2,
"expected 2 deployment callback jobs in v2_job_queue (one per promotion repo), got {} queued + {:?} completed",
last_jobs.len(),
skipped,
);
// Per-repo args sanity check: the two jobs must target different repos.
let mut repo_paths: Vec<String> = last_jobs
.iter()
.filter_map(|j| {
j.args
.as_ref()
.and_then(|a| a.get("repo_url_resource_path"))
.and_then(|v| v.as_str())
.map(|s| s.to_string())
})
.collect();
repo_paths.sort();
repo_paths.dedup();
assert_eq!(
repo_paths.len(),
2,
"expected callbacks to target two distinct repos, got: {:?}",
last_jobs.iter().map(|j| &j.args).collect::<Vec<_>>()
);
// No callback should have been silently skipped via debouncing collision.
assert!(
skipped.iter().all(|(_, s)| s != "skipped"),
"no deployment callback should be marked skipped, got: {:?}",
skipped,
);
Ok(())
}
/// Promotion mode with group_by_folder: items destined for the same per-folder
/// branch must share one debounce key so they accumulate into a single sync
/// job; scripts in different folders must get distinct keys.
@@ -615,8 +773,9 @@ async fn test_promotion_group_by_folder_debounces_per_folder(
// One in a different folder — should get its own key.
create_test_script(&client, "f/other/gamma").await?;
let expected_grouped = "git_sync:folder:f/grouped";
let expected_other = "git_sync:folder:f/other";
// Keys are namespaced by the repo's resource path.
let expected_grouped = "git_sync:$res:u/test-user/test_git_repo:folder:f/grouped";
let expected_other = "git_sync:$res:u/test-user/test_git_repo:folder:f/other";
// Wait for BOTH folder keys to appear, not just the first one.
let keys = wait_for_debounce_key(&db, expected_other, Duration::from_secs(5)).await?;
assert!(
@@ -629,7 +788,9 @@ async fn test_promotion_group_by_folder_debounces_per_folder(
);
// Paths within the same folder must NOT leak as their own keys.
assert!(
!keys.iter().any(|k| k.starts_with("git_sync:script:")),
!keys
.iter()
.any(|k| k.contains(":script:f/grouped/") || k.contains(":script:f/other/")),
"group_by_folder mode should not emit per-path keys, got: {keys:?}"
);
+25 -13
View File
@@ -980,12 +980,10 @@ async fn create_script_internal<'c>(
.fetch_one(&mut *tx)
.await?;
}
let clashing_script = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(
&format!(
"SELECT {} FROM script WHERE path = $1 AND archived = false AND workspace_id = $2",
windmill_common::scripts::SCRIPT_COLUMNS,
),
)
let clashing_script = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(&format!(
"SELECT {} FROM script WHERE path = $1 AND archived = false AND workspace_id = $2",
windmill_common::scripts::SCRIPT_COLUMNS,
))
.bind(&ns.path)
.bind(&w_id)
.fetch_optional(&mut *tx)
@@ -1218,6 +1216,22 @@ async fn create_script_internal<'c>(
}
};
// Failure, Trigger, and Approval scripts are runnable entrypoints by
// definition. They must never be marked `auto_kind = 'lib'`, or they
// disappear from the flow error-handler / trigger / approval pickers
// (which filter out lib scripts). Strip a stray `lib` here so a parser
// misclassification — e.g. failing to detect `main` after a deno_ast
// bump — cannot orphan these scripts in the UI.
let auto_kind = if matches!(
ns.kind,
Some(ScriptKind::Failure) | Some(ScriptKind::Trigger) | Some(ScriptKind::Approval)
) && auto_kind.as_deref() == Some("lib")
{
None
} else {
auto_kind
};
let ci_test_refs =
windmill_common::schema::parse_ci_test_annotation(&ns.content, &lang.as_comment_lit());
// `pipeline` wins over `test` and any client-supplied auto_kind. The
@@ -2298,7 +2312,7 @@ async fn exists_script_by_path(
let path = path.to_path();
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
"SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1)",
path,
w_id
)
@@ -2332,12 +2346,10 @@ async fn get_script_by_hash_internal<'c>(
.fetch_optional(&mut **db)
.await?
} else {
sqlx::query_as::<_, ScriptWithStarred<ScriptRunnableSettingsHandle>>(
&format!(
"SELECT {}, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2",
windmill_common::scripts::SCRIPT_COLUMNS,
),
)
sqlx::query_as::<_, ScriptWithStarred<ScriptRunnableSettingsHandle>>(&format!(
"SELECT {}, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2",
windmill_common::scripts::SCRIPT_COLUMNS,
))
.bind(hash)
.bind(workspace_id)
.fetch_optional(&mut **db)
+23 -9
View File
@@ -210,6 +210,8 @@ pub struct GlobalUserInfo {
first_time_user: bool,
role_source: String,
disabled: bool,
#[serde(skip_serializing_if = "Option::is_none")]
workspace_id: Option<String>,
}
#[derive(Serialize, Debug)]
@@ -297,6 +299,7 @@ pub struct TruncatedToken {
pub last_used_at: chrono::DateTime<chrono::Utc>,
pub scopes: Option<Vec<String>>,
pub workspace_id: Option<String>,
pub read_only: bool,
}
// NewToken is re-exported from windmill-api-auth above
@@ -450,13 +453,17 @@ async fn list_users_as_super_admin(
let rows = if active_only.is_some_and(|x| x) {
sqlx::query_as!(
GlobalUserInfo,
"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
r#"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)
SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source, disabled
SELECT email as "email!", (email NOT IN (SELECT email FROM authors)) as operator_only, login_type::text, verified as "verified!", super_admin as "super_admin!", devops as "devops!", name, company, username, first_time_user as "first_time_user!", role_source as "role_source!", disabled as "disabled!", NULL::text as workspace_id
FROM password
WHERE email IN (SELECT email FROM active_users)
ORDER BY super_admin DESC, devops DESC
LIMIT $1 OFFSET $2",
UNION ALL
SELECT email as "email!", true as operator_only, 'service_account'::text as login_type, true as "verified!", false as "super_admin!", false as "devops!", NULL::text as name, NULL::text as company, username, false as "first_time_user!", 'service_account'::text as "role_source!", disabled as "disabled!", workspace_id
FROM usr
WHERE is_service_account IS true
ORDER BY "super_admin!" DESC, "devops!" DESC
LIMIT $1 OFFSET $2"#,
per_page as i32,
offset as i32
)
@@ -465,8 +472,13 @@ async fn list_users_as_super_admin(
} else {
sqlx::query_as!(
GlobalUserInfo,
"SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \
$1 OFFSET $2",
r#"SELECT email as "email!", login_type::text, verified as "verified!", super_admin as "super_admin!", devops as "devops!", name, company, username, NULL::bool as operator_only, first_time_user as "first_time_user!", role_source as "role_source!", disabled as "disabled!", NULL::text as workspace_id FROM password
UNION ALL
SELECT email as "email!", 'service_account'::text as login_type, true as "verified!", false as "super_admin!", false as "devops!", NULL::text as name, NULL::text as company, username, true as operator_only, false as "first_time_user!", 'service_account'::text as "role_source!", disabled as "disabled!", workspace_id
FROM usr
WHERE is_service_account IS true
ORDER BY "super_admin!" DESC, "devops!" DESC, "email!"
LIMIT $1 OFFSET $2"#,
per_page as i32,
offset as i32
)
@@ -715,7 +727,7 @@ async fn global_whoami(
) -> JsonResult<GlobalUserInfo> {
let user = sqlx::query_as!(
GlobalUserInfo,
"SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password WHERE \
"SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled, NULL::text as workspace_id FROM password WHERE \
email = $1",
email
)
@@ -739,6 +751,7 @@ async fn global_whoami(
first_time_user: false,
role_source: "manual".to_string(),
disabled: false,
workspace_id: None,
}))
} else {
// Service accounts don't have a password row
@@ -755,6 +768,7 @@ async fn global_whoami(
first_time_user: false,
role_source: "service_account".to_string(),
disabled: false,
workspace_id: None,
}))
}
}
@@ -2249,7 +2263,7 @@ async fn list_tokens(
sqlx::query_as!(
TruncatedToken,
"SELECT label, token_prefix, expiration, created_at, \
last_used_at, scopes, workspace_id FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)
last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)
ORDER BY created_at DESC LIMIT $2 OFFSET $3",
email,
per_page as i64,
@@ -2261,7 +2275,7 @@ async fn list_tokens(
sqlx::query_as!(
TruncatedToken,
"SELECT label, token_prefix, expiration, created_at, \
last_used_at, scopes, workspace_id FROM token WHERE email = $1
last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1
ORDER BY created_at DESC LIMIT $2 OFFSET $3",
email,
per_page as i64,
+14 -3
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.699.0
version: 1.702.1
title: Windmill API
contact:
@@ -22631,10 +22631,13 @@ components:
type: string
workspace_id:
type: string
read_only:
type: boolean
required:
- token_prefix
- created_at
- last_used_at
- read_only
ExternalJwtToken:
type: object
@@ -22683,6 +22686,12 @@ components:
type: string
workspace_id:
type: string
read_only:
type: boolean
description: |
If true, the token is restricted to read-only HTTP methods
(GET/HEAD/OPTIONS). Mutating endpoints and job-run actions are
rejected with 403, regardless of the scopes attached.
NewTokenImpersonate:
type: object
@@ -26355,7 +26364,7 @@ components:
type: string
login_type:
type: string
enum: ["password", "github"]
enum: ["password", "github", "service_account"]
super_admin:
type: boolean
devops:
@@ -26374,9 +26383,11 @@ components:
type: boolean
role_source:
type: string
enum: ["manual", "instance_group"]
enum: ["manual", "instance_group", "service_account"]
disabled:
type: boolean
workspace_id:
type: string
required:
- email
+1
View File
@@ -330,6 +330,7 @@ async fn inject_agent_authed(
scopes: None,
username_override: None,
token_prefix: None,
read_only: false,
},
job_id: None,
});
+8 -65
View File
@@ -7,6 +7,7 @@ use async_trait::async_trait;
use serde_json::Value;
use std::collections::HashMap;
use windmill_common::{db::UserDB, utils::StripPath, DB};
use windmill_mcp::common::schema::enrich_resource_schemas;
use windmill_mcp::common::transform::apply_key_transformation;
use windmill_mcp::common::types::{
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo,
@@ -203,71 +204,13 @@ impl McpBackend for WindmillBackend {
schema_obj.properties.insert(new_key, value);
}
for (_key, prop_value) in schema_obj.properties.iter_mut() {
if let Value::Object(prop_map) = prop_value {
if let Some(format_value) = prop_map.get("format") {
if let Value::String(format_str) = format_value {
if format_str.starts_with("resource-") {
let resource_type_key =
format_str.split("-").last().unwrap_or_default().to_string();
let resource_type = resources_types
.iter()
.find(|rt| rt.name == resource_type_key);
let resource_type_obj = resource_type.cloned();
if let Some(resource_cache) = resources_cache.get(&resource_type_key) {
let resources_count = resource_cache.len();
let description = match resource_type_obj {
Some(resource_type_obj) => format!(
"This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}",
resource_type_obj.name,
resource_type_obj.description.as_deref().unwrap_or("No description"),
if resources_count == 0 {
"This resource does not have any available instances, you should create one from your windmill workspace."
} else if resources_count > 1 {
"This resource has multiple available instances, you should precisely select the one you want to use."
} else {
"There is 1 resource available."
}
),
None => "An object parameter.".to_string(),
};
prop_map.insert(
"type".to_string(),
Value::String("string".to_string()),
);
prop_map
.insert("description".to_string(), Value::String(description));
if resources_count > 0 {
let resources_description = resource_cache
.iter()
.map(|resource| {
format!(
"{}: $res:{}",
resource
.description
.as_deref()
.unwrap_or("No title"),
resource.path
)
})
.collect::<Vec<String>>()
.join("\\n");
prop_map.insert(
"description".to_string(),
Value::String(format!(
"{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}",
prop_map.get("description").unwrap_or(&Value::String("No description".to_string())),
resources_description
)),
);
}
}
}
}
}
}
// Enrich every resource reference in the schema — including those
// inside `items`, nested `properties`, etc. — with a description
// listing the available resources. Both shapes are handled:
// { type: "object", format: "resource-<name>" } (top-level scalar)
// { type: "resource", resourceType: "<name>" } (inside list items)
for prop_value in schema_obj.properties.values_mut() {
enrich_resource_schemas(prop_value, resources_cache, resources_types);
}
schema_obj
+151 -52
View File
@@ -110,6 +110,12 @@ struct ScriptMetadata {
pub debouncing_settings: DebouncingSettings,
#[serde(skip_serializing_if = "Option::is_none")]
pub labels: Option<Vec<String>>,
#[serde(skip_serializing_if = "is_empty_extra_perms")]
pub extra_perms: serde_json::Value,
}
fn is_empty_extra_perms(value: &serde_json::Value) -> bool {
value.as_object().is_some_and(|o| o.is_empty()) || value.is_null()
}
pub fn is_none_or_false(val: &Option<bool>) -> bool {
@@ -224,12 +230,34 @@ pub(crate) struct ArchiveQueryParams {
default_ts: Option<String>,
/// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format
settings_version: Option<String>,
/// Opt-in: include `extra_perms` on flow / script / app rows. Default `false`
/// so cross-workspace tarball imports do not carry over ACLs referring to
/// identities that may not exist in the target workspace. `wmill sync pull`
/// passes `true` to surface ACLs in the git-tracked yaml.
preserve_extra_perms: Option<bool>,
}
/// How to handle `extra_perms` in the serialized output.
///
/// * `Drop` — strip the field unconditionally (legacy behavior for
/// types that have never carried ACLs in source).
/// * `KeepEvenEmpty` — always keep the field, even when `{}`. Matches the
/// pre-existing serialization for folders and groups so
/// no customer sees a one-time noisy diff on upgrade.
/// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}`
/// or null. New surface for flow / script / app, which
/// never carried ACLs in source before this change.
#[derive(Clone, Copy)]
pub enum ExtraPermsBehavior {
Drop,
KeepEvenEmpty,
KeepIfNonEmpty,
}
#[inline]
pub fn to_string_without_metadata<T>(
value: &T,
preserve_extra_perms: bool,
extra_perms: ExtraPermsBehavior,
ignore_keys: Option<Vec<&str>>,
) -> Result<String>
where
@@ -274,8 +302,19 @@ where
o2.remove("on_behalf_of");
o2.remove("on_behalf_of_email");
}
if !preserve_extra_perms && obj.contains_key("extra_perms") {
obj.remove("extra_perms");
if obj.contains_key("extra_perms") {
let is_empty_extra_perms = obj
.get("extra_perms")
.map(|v| v.as_object().is_some_and(|o| o.is_empty()) || v.is_null())
.unwrap_or(true);
let drop = match extra_perms {
ExtraPermsBehavior::Drop => true,
ExtraPermsBehavior::KeepEvenEmpty => false,
ExtraPermsBehavior::KeepIfNonEmpty => is_empty_extra_perms,
};
if drop {
obj.remove("extra_perms");
}
}
if obj
.get("default_permissioned_as")
@@ -442,6 +481,7 @@ pub(crate) async fn tarball_workspace(
include_workspace_dependencies,
default_ts,
settings_version,
preserve_extra_perms,
}): Query<ArchiveQueryParams>,
) -> Result<([(HeaderName, String); 2], impl IntoResponse)> {
tracing::info!(
@@ -452,6 +492,16 @@ pub(crate) async fn tarball_workspace(
skip_resources
);
// Opt-in behavior for surfacing per-resource ACLs on flow/app rows.
// Folder and group rows have always carried `extra_perms` in source and
// continue to do so unconditionally (`KeepEvenEmpty`) so existing
// customer git repos see no one-time noisy diff.
let new_kinds_extra_perms = if preserve_extra_perms.unwrap_or(false) {
ExtraPermsBehavior::KeepIfNonEmpty
} else {
ExtraPermsBehavior::Drop
};
let mut tx = user_db.begin(&authed).await?;
// Source-of-truth check for fork-ness: the workspace's parent_workspace_id
@@ -498,7 +548,8 @@ pub(crate) async fn tarball_workspace(
for folder in folders {
archive
.write_to_archive(
&to_string_without_metadata(&folder, true, None).unwrap(),
&to_string_without_metadata(&folder, ExtraPermsBehavior::KeepEvenEmpty, None)
.unwrap(),
&format!("f/{}/folder.meta.json", folder.name),
)
.await?;
@@ -506,15 +557,13 @@ pub(crate) async fn tarball_workspace(
}
{
let scripts = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(
&format!(
"SELECT {} FROM script as o WHERE workspace_id = $1 AND archived = false
let scripts = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(&format!(
"SELECT {} FROM script as o WHERE workspace_id = $1 AND archived = false
AND (draft_only IS NULL OR draft_only = false)
AND created_at = (select max(created_at) from script where path = o.path AND \
workspace_id = $1)",
windmill_common::scripts::SCRIPT_COLUMNS,
),
)
windmill_common::scripts::SCRIPT_COLUMNS,
))
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
@@ -587,6 +636,15 @@ pub(crate) async fn tarball_workspace(
on_behalf_of_email: script.on_behalf_of_email,
modules: script.modules,
labels: script.labels,
// Same opt-in contract as flow/app: the tarball only surfaces
// ACLs when `?preserve_extra_perms=true`. Passing `Null` lets the
// `is_empty_extra_perms` skip-serializer drop the field entirely.
extra_perms: if matches!(new_kinds_extra_perms, ExtraPermsBehavior::KeepIfNonEmpty)
{
script.extra_perms
} else {
serde_json::Value::Null
},
};
let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
archive
@@ -605,7 +663,8 @@ pub(crate) async fn tarball_workspace(
.await?;
for resource in resources {
let resource_str = &to_string_without_metadata(&resource, false, None).unwrap();
let resource_str =
&to_string_without_metadata(&resource, ExtraPermsBehavior::Drop, None).unwrap();
archive
.write_to_archive(&resource_str, &format!("{}.resource.json", resource.path))
.await?;
@@ -622,7 +681,9 @@ pub(crate) async fn tarball_workspace(
.await?;
for resource_type in resource_types {
let resource_str = &to_string_without_metadata(&resource_type, false, None).unwrap();
let resource_str =
&to_string_without_metadata(&resource_type, ExtraPermsBehavior::Drop, None)
.unwrap();
archive
.write_to_archive(
&resource_str,
@@ -644,7 +705,7 @@ pub(crate) async fn tarball_workspace(
.await?;
for flow in flows {
let flow_str = &to_string_without_metadata(&flow, false, None).unwrap();
let flow_str = &to_string_without_metadata(&flow, new_kinds_extra_perms, None).unwrap();
archive
.write_to_archive(&flow_str, &format!("{}.flow.json", flow.path))
.await?;
@@ -673,7 +734,8 @@ pub(crate) async fn tarball_workspace(
Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e))
})?);
}
let var_str = &to_string_without_metadata(&var, false, None).unwrap();
let var_str =
&to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap();
archive
.write_to_archive(&var_str, &format!("{}.variable.json", var.path))
.await?;
@@ -693,7 +755,7 @@ pub(crate) async fn tarball_workspace(
.await?;
for app in apps {
let app_str = &to_string_without_metadata(&app, false, None).unwrap();
let app_str = &to_string_without_metadata(&app, new_kinds_extra_perms, None).unwrap();
let kind = if app.raw_app { "raw_app" } else { "app" };
archive
.write_to_archive(&app_str, &format!("{}.{}.json", app.path, kind))
@@ -711,7 +773,7 @@ pub(crate) async fn tarball_workspace(
workspace_dependencies.len()
);
for dep in workspace_dependencies {
// let dep_str = &to_string_without_metadata(&dep, false, None).unwrap();
// let dep_str = &to_string_without_metadata(&dep, ExtraPermsBehavior::Drop, None).unwrap();
let filename = WorkspaceDependencies::to_path(&dep.name, dep.language)?;
tracing::info!(
"Adding workspace dependency: name={:?}, language={:?}, filename={}",
@@ -739,9 +801,12 @@ pub(crate) async fn tarball_workspace(
let schedule_ignore_keys = fork_schedule_ignore_keys(is_fork);
for schedule in schedules {
let app_str =
&to_string_without_metadata(&schedule, false, schedule_ignore_keys.clone())
.unwrap();
let app_str = &to_string_without_metadata(
&schedule,
ExtraPermsBehavior::Drop,
schedule_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(&app_str, &format!("{}.schedule.json", schedule.path))
.await?;
@@ -777,9 +842,12 @@ pub(crate) async fn tarball_workspace(
let http_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in http_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -796,9 +864,12 @@ pub(crate) async fn tarball_workspace(
let websocket_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in websocket_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -815,9 +886,12 @@ pub(crate) async fn tarball_workspace(
let kafka_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in kafka_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -834,9 +908,12 @@ pub(crate) async fn tarball_workspace(
let sqs_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in sqs_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -853,9 +930,12 @@ pub(crate) async fn tarball_workspace(
let gcp_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in gcp_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -872,9 +952,12 @@ pub(crate) async fn tarball_workspace(
let azure_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in azure_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -891,9 +974,12 @@ pub(crate) async fn tarball_workspace(
let nats_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in nats_triggers {
let trigger_str: &String =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str: &String = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -910,9 +996,12 @@ pub(crate) async fn tarball_workspace(
let postgres_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in postgres_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -929,9 +1018,12 @@ pub(crate) async fn tarball_workspace(
let mqtt_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in mqtt_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -948,9 +1040,12 @@ pub(crate) async fn tarball_workspace(
let email_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
for trigger in email_triggers {
let trigger_str =
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
.unwrap();
let trigger_str = &to_string_without_metadata(
&trigger,
ExtraPermsBehavior::Drop,
trigger_ignore_keys.clone(),
)
.unwrap();
archive
.write_to_archive(
&trigger_str,
@@ -978,7 +1073,7 @@ pub(crate) async fn tarball_workspace(
for trigger in native_triggers {
let trigger_str = &to_string_without_metadata(
&trigger,
false,
ExtraPermsBehavior::Drop,
Some(native_ignore_keys.clone()),
)
.unwrap();
@@ -1021,7 +1116,9 @@ pub(crate) async fn tarball_workspace(
disabled: user.disabled,
email: user.email,
};
let user_str = &to_string_without_metadata(&user, false, Some(vec!["email"])).unwrap();
let user_str =
&to_string_without_metadata(&user, ExtraPermsBehavior::Drop, Some(vec!["email"]))
.unwrap();
archive
.write_to_archive(&user_str, &format!("users/{}.user.json", user.email))
.await?;
@@ -1081,7 +1178,9 @@ pub(crate) async fn tarball_workspace(
admins,
};
let group_str = &to_string_without_metadata(&group, true, None).unwrap();
let group_str =
&to_string_without_metadata(&group, ExtraPermsBehavior::KeepEvenEmpty, None)
.unwrap();
archive
.write_to_archive(&group_str, &format!("groups/{}.group.json", group.name))
.await?;
@@ -200,7 +200,15 @@ fn opaque_json_schema(_: &mut schemars::gen::SchemaGenerator) -> schemars::schem
/// Typed global settings with schema validation.
/// Known settings have explicit fields; unknown settings pass through via `extra`.
///
/// `#[serde(remote = "Self")]` turns the derived (de)serializers into inherent
/// associated functions so we can wrap them with the manual `Deserialize` impl
/// below. The wrapper preserves explicit `null` values (which the typed
/// `Option<T>` fields would otherwise silently drop on round-trip through
/// `to_settings_map`) by stashing them in `extra`, where they survive
/// re-serialization and reach `diff_global_settings` as proper deletes.
#[derive(Deserialize, Serialize, Clone, Debug, Default)]
#[serde(remote = "Self")]
#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
pub struct GlobalSettings {
// Numeric settings
@@ -373,6 +381,42 @@ pub struct GlobalSettings {
pub extra: BTreeMap<String, serde_json::Value>,
}
impl Serialize for GlobalSettings {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
Self::serialize(self, serializer)
}
}
impl<'de> Deserialize<'de> for GlobalSettings {
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
// Capture top-level keys explicitly set to `null` so they survive the
// `to_settings_map` round-trip — typed `Option<T>` fields all use
// `skip_serializing_if = "Option::is_none"`, which would otherwise
// silently drop the null. We stash the null entries in `extra`, which
// serializes them back out as `null` for `diff_global_settings` to
// route to deletes.
let mut value = serde_json::Value::deserialize(deserializer)?;
let null_keys: Vec<String> = value
.as_object()
.map(|m| {
m.iter()
.filter_map(|(k, v)| v.is_null().then(|| k.clone()))
.collect()
})
.unwrap_or_default();
if let Some(obj) = value.as_object_mut() {
for k in &null_keys {
obj.remove(k);
}
}
let mut s = Self::deserialize(value).map_err(serde::de::Error::custom)?;
for k in null_keys {
s.extra.insert(k, serde_json::Value::Null);
}
Ok(s)
}
}
impl GlobalSettings {
/// Convert to a flat `BTreeMap` suitable for DB sync.
pub fn to_settings_map(&self) -> BTreeMap<String, serde_json::Value> {
@@ -572,6 +616,12 @@ pub struct OtelTracingProxySettings {
pub enabled: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub enabled_languages: Vec<ScriptLang>,
/// Comma-separated list of host patterns injected as NO_PROXY into jobs so their HTTP
/// clients bypass the local MITM tracing proxy. Independent of the worker's own
/// NO_PROXY env (which governs the proxy's upstream relay). Use this for clients that
/// pin their own CA (kubectl, helm, terraform providers, aws cli for EKS, etc.).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub no_proxy_hosts: Option<String>,
}
/// Script language identifier (for instance config use).
@@ -1930,6 +1980,101 @@ mod tests {
// to_settings_map edge cases
// -----------------------------------------------------------------------
/// Regression test for bulk-endpoint deletion of typed settings.
///
/// A naive `#[derive(Deserialize)]` would route
/// `{"object_store_cache_config": null}` to the typed `Option<...>` field
/// as `None`, and `skip_serializing_if = "Option::is_none"` would then
/// strip it from `to_settings_map`, so `diff_global_settings` (Merge mode)
/// would never see the deletion. The manual `Deserialize` impl on
/// `GlobalSettings` captures explicit top-level nulls into `extra` so they
/// survive the round-trip and reach `diff_global_settings` as proper
/// deletes.
#[test]
fn explicit_null_on_typed_field_survives_round_trip() {
let json = serde_json::json!({
"object_store_cache_config": null,
"secret_backend": null,
"smtp_settings": null,
"base_url": "https://x",
});
let settings: GlobalSettings =
serde_json::from_value(json).expect("null should deserialize");
assert!(settings.object_store_cache_config.is_none());
assert!(settings.secret_backend.is_none());
assert!(settings.smtp_settings.is_none());
assert_eq!(settings.base_url.as_deref(), Some("https://x"));
let map = settings.to_settings_map();
assert_eq!(map["object_store_cache_config"], serde_json::Value::Null);
assert_eq!(map["secret_backend"], serde_json::Value::Null);
assert_eq!(map["smtp_settings"], serde_json::Value::Null);
assert_eq!(map["base_url"], serde_json::json!("https://x"));
}
/// Absent keys remain absent — critical so a PUT that only sets a single
/// field doesn't accidentally delete every other setting in Merge mode.
#[test]
fn absent_typed_fields_do_not_appear_in_map() {
let settings: GlobalSettings =
serde_json::from_value(serde_json::json!({"base_url": "https://x"})).unwrap();
let map = settings.to_settings_map();
assert!(!map.contains_key("object_store_cache_config"));
assert!(!map.contains_key("smtp_settings"));
assert_eq!(map.get("base_url"), Some(&serde_json::json!("https://x")));
}
/// End-to-end: deserialize → `to_settings_map` → diff in Merge mode with
/// an explicit null produces a delete (the scenario that silently failed
/// before the manual `Deserialize` impl).
#[test]
fn deserialize_then_diff_deletes_typed_null() {
let mut current = BTreeMap::new();
current.insert(
"object_store_cache_config".to_string(),
serde_json::json!({"type": "S3", "bucket": "b"}),
);
let desired: GlobalSettings =
serde_json::from_value(serde_json::json!({"object_store_cache_config": null})).unwrap();
let desired_map = desired.to_settings_map();
let diff = diff_global_settings(&current, &desired_map, ApplyMode::Merge);
assert!(diff.upserts.is_empty());
assert_eq!(diff.deletes, vec!["object_store_cache_config".to_string()]);
}
/// Nested nulls inside a typed sub-struct are not promoted to top-level
/// deletes — only the top-level key matters, mirroring the per-key API.
#[test]
fn nested_null_inside_typed_field_is_not_treated_as_top_level_null() {
let settings: GlobalSettings =
serde_json::from_value(serde_json::json!({"smtp_settings": {"smtp_host": null}}))
.unwrap();
let map = settings.to_settings_map();
assert!(
map["smtp_settings"].is_object(),
"smtp_settings should be an object, not null"
);
}
/// Unknown/extra keys with explicit null still flow through `extra` — this
/// was already correct before the manual impl; guard against regression.
#[test]
fn explicit_null_on_extra_field_survives_round_trip() {
let settings: GlobalSettings =
serde_json::from_value(serde_json::json!({"unknown_legacy_setting": null})).unwrap();
let map = settings.to_settings_map();
assert_eq!(map["unknown_legacy_setting"], serde_json::Value::Null);
}
/// Top-level non-object input must reject with a deserialize error rather
/// than silently producing defaults. Matches the previous derive behavior.
#[test]
fn non_object_top_level_input_errors() {
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!(null)).is_err());
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!("s")).is_err());
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!(42)).is_err());
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!([])).is_err());
}
#[test]
fn to_settings_map_empty_defaults() {
let settings = GlobalSettings::default();
+431 -17
View File
@@ -2,11 +2,11 @@
//!
//! Contains functions for converting Windmill schemas into MCP-compatible formats.
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use serde_json::Value;
use serde_json::{Map, Value};
use super::types::SchemaType;
use super::types::{ResourceInfo, ResourceType, SchemaType};
use windmill_common::scripts::Schema;
/// Convert a Windmill Schema to a SchemaType
@@ -22,38 +22,195 @@ pub fn convert_schema_to_schema_type(schema: Option<Schema>) -> SchemaType {
schema_obj
}
/// Extract resource type keys from a schema
///
/// Scans the schema properties for fields with format "resource-{type}"
/// and returns a set of all unique resource type names found.
pub fn extract_resource_types_from_schema(schema: &SchemaType) -> HashSet<String> {
let mut resource_types = HashSet::new();
for (_key, prop_value) in schema.properties.iter() {
if let Value::Object(prop_map) = prop_value {
if let Some(Value::String(format_str)) = prop_map.get("format") {
if let Some(rt) = format_str.strip_prefix("resource-") {
resource_types.insert(rt.to_string());
}
/// If `node` is a schema describing a Windmill resource reference, return the
/// resource type name. Recognizes both on-disk shapes:
/// - Form A (top-level scalar): `{ type: "object", format: "resource-<name>" }`
/// - Form B (inside `items`): `{ type: "resource", resourceType: "<name>" }`
fn resource_type_of_schema_node(node: &Value) -> Option<String> {
let obj = node.as_object()?;
if let Some(Value::String(fmt)) = obj.get("format") {
if let Some(name) = fmt.strip_prefix("resource-") {
return Some(name.to_string());
}
}
if obj.get("type").and_then(Value::as_str) == Some("resource") {
if let Some(Value::String(name)) = obj.get("resourceType") {
return Some(name.clone());
}
}
None
}
/// Recursively collect resource type names referenced at any depth in `node`.
fn collect_resource_types(node: &Value, out: &mut HashSet<String>) {
if let Some(rt) = resource_type_of_schema_node(node) {
out.insert(rt);
}
let Some(obj) = node.as_object() else { return };
if let Some(Value::Object(props)) = obj.get("properties") {
for v in props.values() {
collect_resource_types(v, out);
}
}
if let Some(items) = obj.get("items") {
collect_resource_types(items, out);
}
if let Some(additional) = obj.get("additionalProperties") {
if additional.is_object() {
collect_resource_types(additional, out);
}
}
for kw in ["allOf", "oneOf", "anyOf"] {
if let Some(Value::Array(arr)) = obj.get(kw) {
for s in arr {
collect_resource_types(s, out);
}
}
}
}
/// Extract resource type keys referenced anywhere in a schema (top-level
/// properties, nested objects, array items, additionalProperties, and
/// allOf/oneOf/anyOf subschemas). Recognizes both the `format: resource-<name>`
/// and `type: resource` + `resourceType` shapes.
pub fn extract_resource_types_from_schema(schema: &SchemaType) -> HashSet<String> {
let mut resource_types = HashSet::new();
for prop_value in schema.properties.values() {
collect_resource_types(prop_value, &mut resource_types);
}
resource_types
}
/// Rewrite a single schema node that points to a Windmill resource: set
/// `type: "string"` and inject a description listing the available resources.
/// Mirrors the top-level behavior that used to live in
/// `transform_schema_for_resources`. No-op if the resource type isn't in the
/// pre-fetched cache.
fn apply_resource_enrichment(
prop_map: &mut Map<String, Value>,
resource_type_key: &str,
resources_cache: &HashMap<String, Vec<ResourceInfo>>,
resources_types: &[ResourceType],
) {
let Some(resource_cache) = resources_cache.get(resource_type_key) else {
return;
};
let resource_type = resources_types
.iter()
.find(|rt| rt.name == resource_type_key);
let resources_count = resource_cache.len();
let description = match resource_type {
Some(rt) => format!(
"This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}",
rt.name,
rt.description.as_deref().unwrap_or("No description"),
if resources_count == 0 {
"This resource does not have any available instances, you should create one from your windmill workspace."
} else if resources_count > 1 {
"This resource has multiple available instances, you should precisely select the one you want to use."
} else {
"There is 1 resource available."
}
),
None => "An object parameter.".to_string(),
};
prop_map.insert("type".to_string(), Value::String("string".to_string()));
prop_map.insert("description".to_string(), Value::String(description));
// Drop the Windmill-internal keys we just consumed so the node is clean
// regardless of whether `make_schema_compatible` runs after us. (Its strip
// only fires while `type == "resource"`, which is no longer true here.)
prop_map.remove("resourceType");
if prop_map
.get("format")
.and_then(Value::as_str)
.is_some_and(|s| s.starts_with("resource-"))
{
prop_map.remove("format");
}
if resources_count > 0 {
let resources_description = resource_cache
.iter()
.map(|resource| {
format!(
"{}: $res:{}",
resource.description.as_deref().unwrap_or("No title"),
resource.path
)
})
.collect::<Vec<String>>()
.join("\\n");
let prior_description = prop_map
.get("description")
.and_then(Value::as_str)
.unwrap_or("No description")
.to_string();
prop_map.insert(
"description".to_string(),
Value::String(format!(
"{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}",
prior_description, resources_description
)),
);
}
}
/// Walk a schema and enrich every Windmill-resource reference (in either shape,
/// at any nesting depth) with `type: "string"` and a description listing
/// available resources. The non-standard keys (`format: resource-*`,
/// `resourceType`) consumed by the enrichment are stripped in place.
pub fn enrich_resource_schemas(
node: &mut Value,
resources_cache: &HashMap<String, Vec<ResourceInfo>>,
resources_types: &[ResourceType],
) {
if let Some(rt_key) = resource_type_of_schema_node(node) {
if let Value::Object(obj) = node {
apply_resource_enrichment(obj, &rt_key, resources_cache, resources_types);
}
}
let Some(obj) = node.as_object_mut() else {
return;
};
if let Some(Value::Object(props)) = obj.get_mut("properties") {
for v in props.values_mut() {
enrich_resource_schemas(v, resources_cache, resources_types);
}
}
if let Some(items) = obj.get_mut("items") {
enrich_resource_schemas(items, resources_cache, resources_types);
}
if let Some(additional) = obj.get_mut("additionalProperties") {
if additional.is_object() {
enrich_resource_schemas(additional, resources_cache, resources_types);
}
}
for kw in ["allOf", "oneOf", "anyOf"] {
if let Some(Value::Array(arr)) = obj.get_mut(kw) {
for s in arr.iter_mut() {
enrich_resource_schemas(s, resources_cache, resources_types);
}
}
}
}
/// Transform a JSON schema for maximum MCP client compatibility.
///
/// Ensures schemas conform to JSON Schema draft 2020-12 by:
/// - Converting `integer` type to `number` (some clients don't support integer)
/// - Removing invalid non-array `enum` values
/// - Stripping non-standard keywords (`originalType`, `format` with `resource-*` prefix)
/// - Rewriting the Windmill pseudo-type `type: "resource"` to `type: "string"`
/// - Fixing contradictory schemas (`type: "string"` with `properties` → `type: "object"`)
/// - Removing `default: null` when the type doesn't include `null`
/// - Adding `type: "object"` to empty schemas that have no type
pub fn make_schema_compatible(schema: &mut Value) {
let Value::Object(obj) = schema else { return };
// 1. Strip non-standard keywords that aren't part of JSON Schema
// 1. Strip non-standard keywords that aren't part of JSON Schema. The
// Windmill-internal `resourceType` is dropped unconditionally so it can't
// leak through even on enrichment cache-miss paths.
obj.remove("originalType");
obj.remove("resourceType");
// 2. Strip non-standard format values (resource-* is Windmill-internal)
if obj
@@ -64,6 +221,14 @@ pub fn make_schema_compatible(schema: &mut Value) {
obj.remove("format");
}
// 2b. Rewrite Windmill pseudo-type `resource` to `string`. The parser emits
// this shape (with a sibling `resourceType` key) for `list[ResourceType]`
// params; "resource" is not in the JSON Schema 2020-12 type enum and is
// rejected by strict validators (e.g. Anthropic's tool registration).
if obj.get("type").and_then(|v| v.as_str()) == Some("resource") {
obj.insert("type".to_string(), Value::String("string".to_string()));
}
// 3. Fix contradictory type: if `properties` is present, type must be "object"
if obj.contains_key("properties") {
match obj.get("type").and_then(|v| v.as_str()) {
@@ -151,8 +316,27 @@ pub fn make_schema_compatible(schema: &mut Value) {
#[cfg(test)]
mod tests {
use super::make_schema_compatible;
use super::*;
use crate::common::types::{ResourceInfo, ResourceType};
use serde_json::json;
use std::collections::HashMap;
fn aws_resources() -> (HashMap<String, Vec<ResourceInfo>>, Vec<ResourceType>) {
let mut cache = HashMap::new();
cache.insert(
"c_aws_account".to_string(),
vec![ResourceInfo {
path: "f/platform/aws_dev".to_string(),
description: Some("Dev account".to_string()),
resource_type: "c_aws_account".to_string(),
}],
);
let types = vec![ResourceType {
name: "c_aws_account".to_string(),
description: Some("AWS account".to_string()),
}];
(cache, types)
}
#[test]
fn converts_nested_integer_types() {
@@ -356,6 +540,96 @@ mod tests {
assert_eq!(schema, json!({}));
}
#[test]
fn rewrites_resource_pseudo_type_to_string() {
let mut schema = json!({
"type": "resource",
"resourceType": "c_aws_account"
});
make_schema_compatible(&mut schema);
assert_eq!(schema["type"], json!("string"));
assert!(schema.get("resourceType").is_none());
}
#[test]
fn rewrites_resource_pseudo_type_inside_array_items() {
// Phocas repro: list[ResourceType] parameter. Anthropic rejected this
// with "tools.<N>.custom.input_schema: JSON schema is invalid" because
// "resource" is not in the draft 2020-12 type enum.
let mut schema = json!({
"type": "object",
"properties": {
"accounts": {
"type": "array",
"items": {
"type": "resource",
"resourceType": "c_aws_account"
}
}
}
});
make_schema_compatible(&mut schema);
assert_eq!(
schema["properties"]["accounts"]["items"]["type"],
json!("string")
);
assert!(schema["properties"]["accounts"]["items"]
.get("resourceType")
.is_none());
}
#[test]
fn rewrites_resource_pseudo_type_inside_nested_properties() {
let mut schema = json!({
"type": "object",
"properties": {
"config": {
"type": "object",
"properties": {
"db": {
"type": "resource",
"resourceType": "postgresql"
}
}
}
}
});
make_schema_compatible(&mut schema);
assert_eq!(
schema["properties"]["config"]["properties"]["db"]["type"],
json!("string")
);
assert!(schema["properties"]["config"]["properties"]["db"]
.get("resourceType")
.is_none());
}
#[test]
fn strips_nested_resource_format_inside_array_items() {
let mut schema = json!({
"type": "object",
"properties": {
"dbs": {
"type": "array",
"items": {
"type": "object",
"format": "resource-postgresql"
}
}
}
});
make_schema_compatible(&mut schema);
assert!(schema["properties"]["dbs"]["items"].get("format").is_none());
}
#[test]
fn adds_type_to_schema_with_properties_but_no_type() {
let mut schema = json!({
@@ -368,4 +642,144 @@ mod tests {
assert_eq!(schema["type"], json!("object"));
}
#[test]
fn extract_resource_types_top_level_form_a() {
let schema: SchemaType = serde_json::from_value(json!({
"type": "object",
"properties": {
"db": { "type": "object", "format": "resource-postgresql" }
},
"required": []
}))
.unwrap();
let types = extract_resource_types_from_schema(&schema);
assert!(types.contains("postgresql"));
assert_eq!(types.len(), 1);
}
#[test]
fn extract_resource_types_inside_array_items_form_b() {
let schema: SchemaType = serde_json::from_value(json!({
"type": "object",
"properties": {
"accounts": {
"type": "array",
"items": { "type": "resource", "resourceType": "c_aws_account" }
}
},
"required": []
}))
.unwrap();
let types = extract_resource_types_from_schema(&schema);
assert!(types.contains("c_aws_account"));
}
#[test]
fn extract_resource_types_inside_nested_properties_and_one_of() {
let schema: SchemaType = serde_json::from_value(json!({
"type": "object",
"properties": {
"config": {
"type": "object",
"properties": {
"db": { "type": "resource", "resourceType": "postgresql" }
}
},
"either": {
"oneOf": [
{ "type": "object", "format": "resource-mysql" },
{ "type": "string" }
]
}
},
"required": []
}))
.unwrap();
let types = extract_resource_types_from_schema(&schema);
assert!(types.contains("postgresql"));
assert!(types.contains("mysql"));
}
#[test]
fn enrich_top_level_form_a_resource() {
let (cache, types) = aws_resources();
let mut node = json!({
"type": "object",
"format": "resource-c_aws_account"
});
enrich_resource_schemas(&mut node, &cache, &types);
assert_eq!(node["type"], json!("string"));
assert!(node.get("format").is_none());
let desc = node["description"].as_str().unwrap();
assert!(desc.contains("c_aws_account"));
assert!(desc.contains("$res:f/platform/aws_dev"));
}
#[test]
fn enrich_inside_array_items_form_b() {
// Phocas repro: items use the parser-style `type: resource` form.
let (cache, types) = aws_resources();
let mut node = json!({
"type": "array",
"items": { "type": "resource", "resourceType": "c_aws_account" }
});
enrich_resource_schemas(&mut node, &cache, &types);
// The items schema should be rewritten to string with a description
// listing the available resources, and the Windmill-internal
// resourceType key should be stripped.
assert_eq!(node["items"]["type"], json!("string"));
assert!(node["items"].get("resourceType").is_none());
let desc = node["items"]["description"].as_str().unwrap();
assert!(desc.contains("$res:f/platform/aws_dev"));
}
#[test]
fn enrich_is_noop_when_resource_type_not_in_cache() {
let mut node = json!({
"type": "object",
"format": "resource-unknown_type"
});
let before = node.clone();
enrich_resource_schemas(&mut node, &HashMap::new(), &[]);
assert_eq!(node, before);
}
#[test]
fn enrich_deeply_nested_resource() {
let (cache, types) = aws_resources();
let mut node = json!({
"type": "object",
"properties": {
"outer": {
"type": "object",
"properties": {
"inner": {
"type": "array",
"items": {
"type": "resource",
"resourceType": "c_aws_account"
}
}
}
}
}
});
enrich_resource_schemas(&mut node, &cache, &types);
assert_eq!(
node["properties"]["outer"]["properties"]["inner"]["items"]["type"],
json!("string")
);
}
}
@@ -33,6 +33,13 @@ pub trait McpAuth: Send + Sync + Clone + 'static {
/// Get token scopes
fn scopes(&self) -> Option<&[String]>;
/// True if the token was created with the `read_only` flag.
/// When set, write-capable tools must be hidden from `list_tools` and
/// rejected by `call_tool`. Defaults to false so existing impls compile.
fn read_only(&self) -> bool {
false
}
/// Check if the user has an MCP scope
fn has_mcp_scope(&self) -> bool {
self.scopes()
@@ -26,6 +26,12 @@ pub struct EndpointTool {
pub body_field_renames: Option<serde_json::Value>,
}
/// True if this endpoint is safe to expose to a read-only token. Mirrors the
/// `read_only_hint` computed by `create_endpoint_annotations`: only `GET`.
pub fn is_endpoint_read_only(tool: &EndpointTool) -> bool {
tool.method.as_ref() == "GET"
}
/// Convert a single endpoint tool to MCP tool
pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool {
let mut combined_properties = serde_json::Map::new();
+1 -1
View File
@@ -12,7 +12,7 @@ pub mod tools;
// Re-export main types
pub use backend::{BackendResult, McpAuth, McpBackend};
pub use endpoints::{endpoint_tool_to_mcp_tool, EndpointTool};
pub use endpoints::{endpoint_tool_to_mcp_tool, is_endpoint_read_only, EndpointTool};
pub use runner::Runner;
pub use tools::create_tool_from_item;
+114 -83
View File
@@ -145,94 +145,100 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
let favorites_only = scope_config.favorites;
// Fetch all items concurrently
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
self.backend
.list_scripts(&auth, &workspace_id, favorites_only, None),
self.backend
.list_flows(&auth, &workspace_id, favorites_only, None),
self.backend.list_resource_types(&auth, &workspace_id),
async {
if let Some(ref apps) = scope_config.hub_apps {
self.backend.list_hub_scripts(Some(apps)).await
} else {
Ok(vec![])
}
}
)?;
// Filter items based on scope
let filtered_scripts: Vec<_> = scripts
.into_iter()
.filter(|s| !scope_config.granular || scope_config.is_allowed("script", &s.path))
.collect();
let filtered_flows: Vec<_> = flows
.into_iter()
.filter(|f| !scope_config.granular || scope_config.is_allowed("flow", &f.path))
.collect();
// Collect all needed resource types from all schemas
let mut needed_resource_types: HashSet<String> = HashSet::new();
for script in &filtered_scripts {
needed_resource_types.extend(extract_resource_types_from_schema(&script.get_schema()));
}
for flow in &filtered_flows {
needed_resource_types.extend(extract_resource_types_from_schema(&flow.get_schema()));
}
for hub_script in &hub_scripts {
needed_resource_types
.extend(extract_resource_types_from_schema(&hub_script.get_schema()));
}
// Pre-fetch all resources
let resource_futures: Vec<_> = needed_resource_types
.into_iter()
.map(|rt| {
let backend = self.backend.clone();
let auth = auth.clone();
let workspace_id = workspace_id.clone();
async move {
backend
.list_resources(&auth, &workspace_id, &rt)
.await
.map(|resources| (rt, resources))
}
})
.collect();
let resource_results = futures::future::try_join_all(resource_futures).await?;
let resources_cache: HashMap<String, Vec<ResourceInfo>> =
resource_results.into_iter().collect();
let read_only = auth.read_only();
let mut tools = Vec::new();
for script in &filtered_scripts {
tools.push(create_tool_from_item(
script,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
// Read-only tokens cannot run scripts/flows/hub-scripts (running is a
// mutating action), so skip the script/flow/hub/resource fetches
// entirely — they would only be discarded below.
if !read_only {
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
self.backend
.list_scripts(&auth, &workspace_id, favorites_only, None),
self.backend
.list_flows(&auth, &workspace_id, favorites_only, None),
self.backend.list_resource_types(&auth, &workspace_id),
async {
if let Some(ref apps) = scope_config.hub_apps {
self.backend.list_hub_scripts(Some(apps)).await
} else {
Ok(vec![])
}
}
)?;
for flow in &filtered_flows {
tools.push(create_tool_from_item(
flow,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
let filtered_scripts: Vec<_> = scripts
.into_iter()
.filter(|s| !scope_config.granular || scope_config.is_allowed("script", &s.path))
.collect();
for hub_script in &hub_scripts {
tools.push(create_tool_from_item(
hub_script,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
let filtered_flows: Vec<_> = flows
.into_iter()
.filter(|f| !scope_config.granular || scope_config.is_allowed("flow", &f.path))
.collect();
// Collect all needed resource types from all schemas
let mut needed_resource_types: HashSet<String> = HashSet::new();
for script in &filtered_scripts {
needed_resource_types
.extend(extract_resource_types_from_schema(&script.get_schema()));
}
for flow in &filtered_flows {
needed_resource_types
.extend(extract_resource_types_from_schema(&flow.get_schema()));
}
for hub_script in &hub_scripts {
needed_resource_types
.extend(extract_resource_types_from_schema(&hub_script.get_schema()));
}
// Pre-fetch all resources
let resource_futures: Vec<_> = needed_resource_types
.into_iter()
.map(|rt| {
let backend = self.backend.clone();
let auth = auth.clone();
let workspace_id = workspace_id.clone();
async move {
backend
.list_resources(&auth, &workspace_id, &rt)
.await
.map(|resources| (rt, resources))
}
})
.collect();
let resource_results = futures::future::try_join_all(resource_futures).await?;
let resources_cache: HashMap<String, Vec<ResourceInfo>> =
resource_results.into_iter().collect();
for script in &filtered_scripts {
tools.push(create_tool_from_item(
script,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
for flow in &filtered_flows {
tools.push(create_tool_from_item(
flow,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
for hub_script in &hub_scripts {
tools.push(create_tool_from_item(
hub_script,
self.backend.as_ref(),
&resources_cache,
&resource_types,
));
}
}
// Add endpoint tools from the generated MCP tools, filtered by scope
@@ -241,6 +247,9 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) {
continue;
}
if read_only && !crate::server::is_endpoint_read_only(&endpoint_tool) {
continue;
}
tools.push(endpoint_tool_to_mcp_tool(&endpoint_tool));
}
@@ -259,6 +268,7 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
let scopes = auth.scopes().unwrap_or(&[]);
let scope_config =
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
let read_only = auth.read_only();
let args = request.arguments.map(Value::Object).unwrap_or(Value::Null);
@@ -278,6 +288,15 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
None,
));
}
if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) {
return Err(ErrorData::internal_error(
format!(
"Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations",
endpoint_tool.name
),
None,
));
}
// This is an endpoint tool, call via backend
let result = self
@@ -294,6 +313,18 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
}
}
// Anything below this point runs a script or flow, which is a mutating
// action and must be denied for read-only tokens.
if read_only {
return Err(ErrorData::internal_error(
format!(
"Access denied: tool '{}' runs a script/flow and this token is restricted to read-only operations",
request.name
),
None,
));
}
// Resolve the tool name to (type, path, is_hub)
let (type_str, is_hub, is_hashed) = parse_tool_prefix(&request.name).map_err(|e| {
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
@@ -94,6 +94,7 @@ async fn new_webhook_token(
None,
Some(scopes),
Some(workspace_id.to_owned()),
None,
);
let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?;
@@ -0,0 +1,206 @@
//! End-to-end coverage for the EE HTTP-tracing path on nativets.
//!
//! Pairs with `otel_init.rs` (which pins only the `OTEL_GLOBALS`
//! population contract). This test exercises the full chain that
//! actually delivers a span to a collector:
//!
//! `deno_telemetry::init` with EE config (Rust)
//! → `globalThis.__bootstrapOtel()` (JS, flips TRACING_ENABLED)
//! → user `fetch()` → deno_fetch's `builtinTracer().startSpan`
//! → `BatchSpanProcessor` → `HttpExporter` (OTLP/HTTP-binary)
//! → our mock OTLP listener captures the request bytes
//!
//! Without the v1.702.0 fix (#573 EE / #9163 OSS), the third arrow
//! panics in a tokio worker. With the fix in place, the span is
//! emitted and shows up at the listener — which is what the customer
//! is paying for when they enable HTTP tracing on nativets.
//!
//! `#[ignore]`'d: spins a V8 isolate (~seconds) and binds two TCP
//! listeners. Run with `cargo test -p windmill-runtime-nativets
//! --test otel_e2e -- --ignored`.
//!
//! Owns its own test binary so `OTEL_GLOBALS`'s `OnceCell` doesn't
//! race with `otel_init.rs`.
use std::sync::Arc;
use std::time::Duration;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::sync::Mutex;
use windmill_runtime_nativets::{deno_telemetry, transpile_ts, NativeAnnotation, PrewarmedIsolate};
/// Bind 127.0.0.1:0 and spawn an accept loop. Each connection is
/// read until idle/EOF, the body is appended to `captured`, then we
/// respond with HTTP 200. Returns the bound port.
async fn spawn_capturing_http(captured: Arc<Mutex<Vec<Vec<u8>>>>) -> u16 {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
loop {
let Ok((mut sock, _)) = listener.accept().await else {
break;
};
let captured = captured.clone();
tokio::spawn(async move {
let mut buf = Vec::new();
let mut tmp = [0u8; 8192];
let _ = tokio::time::timeout(Duration::from_millis(300), async {
loop {
match sock.read(&mut tmp).await {
Ok(0) | Err(_) => break,
Ok(n) => buf.extend_from_slice(&tmp[..n]),
}
}
})
.await;
let _ = sock
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
.await;
captured.lock().await.push(buf);
});
}
});
port
}
/// Initialize `deno_telemetry` with the exact `OtelConfig` shape that
/// the EE `load_internal_otel_exporter` ships in production. Keeps
/// this test in lockstep with the actual call site: if production
/// drifts away from `tracing_enabled + Capture`, this test breaks
/// before the customer-facing panic does.
fn init_with_ee_otel_config() {
deno_telemetry::init(
deno_telemetry::OtelRuntimeConfig {
runtime_name: "windmill-nativets".into(),
runtime_version: "0".into(),
},
deno_telemetry::OtelConfig {
tracing_enabled: true,
console: deno_telemetry::OtelConsoleConfig::Capture,
..Default::default()
},
)
.expect("deno_telemetry init failed");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[ignore = "spins V8 + tcp listeners; run with --ignored"]
async fn fetch_after_init_otel_emits_span_to_collector() {
let _ = rustls::crypto::ring::default_provider().install_default();
// 1. Stand up two listeners: one that pretends to be the user's
// fetch target, one that pretends to be the OTLP collector.
let fetch_hits: Arc<Mutex<Vec<Vec<u8>>>> = Arc::new(Mutex::new(Vec::new()));
let otlp_hits: Arc<Mutex<Vec<Vec<u8>>>> = Arc::new(Mutex::new(Vec::new()));
let fetch_port = spawn_capturing_http(fetch_hits.clone()).await;
let otlp_port = spawn_capturing_http(otlp_hits.clone()).await;
// 2. Point the deno_telemetry exporter at the mock collector and
// initialize. Mirrors `load_internal_otel_exporter` in EE.
//
// SAFETY: test runs in its own process binary; no other thread
// reads OTEL_EXPORTER_OTLP_ENDPOINT before init returns.
unsafe {
std::env::set_var(
"OTEL_EXPORTER_OTLP_ENDPOINT",
format!("http://127.0.0.1:{otlp_port}"),
);
}
init_with_ee_otel_config();
assert!(
deno_telemetry::OTEL_GLOBALS.get().is_some(),
"OTEL_GLOBALS missing — load_internal_otel_exporter's config regressed?"
);
// 3. Run user TS that bootstraps OTel and then issues a fetch
// at the mock target. `__bootstrapOtel` is fire-and-forget
// (resolves a dynamic import on the microtask queue); the
// `setTimeout` loop yields a few times so the import resolves
// and the `TRACING_ENABLED` flag is set before fetch runs.
let ts = format!(
r#"
declare const globalThis: any;
export async function main(): Promise<number> {{
globalThis.__bootstrapOtel();
// Yield multiple microtask + timer turns so the dynamic import
// in __bootstrapOtel resolves and TRACING_ENABLED flips before
// fetch runs (otherwise deno_fetch skips the span entirely).
for (let i = 0; i < 5; i++) {{
await new Promise<void>(r => setTimeout(r, 10));
}}
const resp = await fetch("http://127.0.0.1:{fetch_port}/probe");
return resp.status;
}}
"#
);
let js = transpile_ts(ts).expect("transpile failed");
let ann = NativeAnnotation { useragent: None, proxy: None };
let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None);
iso.wait_ready().await.expect("isolate failed to pre-warm");
let res = iso
.start_execution("{}".to_string())
.wait()
.await
.expect("isolate panicked");
// The exact bug: pre-fix, fetch panics this isolate. Post-fix,
// we get back the mock target's 200.
let raw = res.result.expect("user script returned an error");
assert_eq!(raw.get(), "200", "fetch should return mock target status");
assert_eq!(
fetch_hits.lock().await.len(),
1,
"fetch target should have been hit exactly once"
);
// 4. Force the BatchSpanProcessor to flush so the exporter posts
// to our mock collector synchronously (default flush interval
// is ~5s; tests can't wait that long).
deno_telemetry::flush();
// Exporter is async over the OTel runtime; give it a beat to
// actually send the HTTP request.
tokio::time::sleep(Duration::from_millis(500)).await;
let otlp_captured = otlp_hits.lock().await;
assert!(
!otlp_captured.is_empty(),
"OTLP collector should have received at least one export — \
spans aren't reaching the collector after init"
);
// Verify the export carries our fetch span. OTLP is protobuf, so
// grep the raw bytes for OTel HTTP semantic-convention markers
// that deno_fetch's auto-instrumentation attaches:
// - the target URL ("url.full" attribute)
// - "http.request.method" attribute
let combined: Vec<u8> = otlp_captured.iter().flatten().copied().collect();
let bytes_contain = |needle: &[u8]| combined.windows(needle.len()).any(|w| w == needle);
let url_marker = format!("http://127.0.0.1:{fetch_port}/probe");
let has_url = bytes_contain(url_marker.as_bytes());
let has_method_attr = bytes_contain(b"http.request.method");
if !(has_url && has_method_attr) {
eprintln!(
"OTLP bytes ({}): {:?}",
combined.len(),
String::from_utf8_lossy(&combined)
);
}
assert!(
has_url,
"exported OTLP body should reference the fetched URL ({}); got {} bytes",
url_marker,
combined.len()
);
assert!(
has_method_attr,
"exported OTLP body should carry HTTP semconv attributes (http.request.method); got {} bytes",
combined.len()
);
}
@@ -0,0 +1,88 @@
//! Regression test for the deno_telemetry 0.31 nativets-fetch panic.
//!
//! Setup: when EE's `load_internal_otel_exporter` runs (HTTP tracing
//! enabled), it calls `deno_telemetry::init` and then flips
//! `DENO_OTEL_INITIALIZED=true` so `js_eval` will run
//! `globalThis.__bootstrapOtel()` inside the nativets JsRuntime. That
//! bootstrap unconditionally sets JS-side `TRACING_ENABLED=true`, which
//! makes `deno_fetch`'s `fetch()` call `builtinTracer().startSpan(...)`
//! — and `OtelTracer::builtin()` does `OTEL_GLOBALS.get().unwrap()`.
//!
//! In deno_telemetry 0.31, `init` was given an early-return guard: if
//! `tracing_enabled`, `metrics_enabled`, and `console` are all
//! off/Ignore, it returns `Ok(())` *without populating OTEL_GLOBALS*.
//! v1.700.0 was passing `OtelConfig::default()` (all off) — so the JS
//! bootstrap proceeded but the Rust-side OnceCell was empty, and the
//! first `fetch()` panicked the tokio worker in a context that cannot
//! unwind. Fixed in v1.702.0 by passing `tracing_enabled: true` +
//! `console: Capture` from `load_internal_otel_exporter` (PRs #573 EE
//! / #9163 OSS), matching the JS bootstrap shape `[1, 0, 1, 0]`.
//!
//! This test pins that contract directly against `deno_telemetry::
//! init` — the function the EE call site invokes — so the next time
//! the dep is bumped and someone is tempted to "simplify" the config
//! back to `OtelConfig::default()`, CI catches it.
//!
//! Lives in `tests/` (not `src/`) so it gets its own test binary and
//! its own process — `OTEL_GLOBALS` is a `OnceCell`, so sharing it
//! with the smoke tests in `src/smoke_tests.rs` would race.
use windmill_runtime_nativets::deno_telemetry;
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn ee_otel_init_config_populates_otel_globals() {
// deno_telemetry::init builds an HttpExporter (uses rustls) — it
// needs a process-wide CryptoProvider, which the real binary
// installs in setup_deno_runtime / main. Mirror that here.
let _ = rustls::crypto::ring::default_provider().install_default();
// Pre-condition: nothing else in this test binary has touched the
// OnceCell, so it must be empty.
assert!(
deno_telemetry::OTEL_GLOBALS.get().is_none(),
"OTEL_GLOBALS must start empty in a fresh test process"
);
// Step 1: reproduce the footgun. `OtelConfig::default()` has
// tracing/metrics off and `console = Ignore`, which trips the
// 0.31 early-return — Ok(()) but OnceCell stays empty.
deno_telemetry::init(
deno_telemetry::OtelRuntimeConfig {
runtime_name: "windmill-nativets-test".into(),
runtime_version: "0".into(),
},
deno_telemetry::OtelConfig::default(),
)
.expect("init with default config returns Ok (early-return)");
assert!(
deno_telemetry::OTEL_GLOBALS.get().is_none(),
"deno_telemetry 0.31 contract: init with all-disabled config \
must NOT populate OTEL_GLOBALS if this changes on a future \
bump, load_internal_otel_exporter can drop the explicit \
tracing_enabled/console fields."
);
// Step 2: re-call init with the exact config shape that the EE
// `load_internal_otel_exporter` ships — this is what production
// hits, so the test exercises the actual prod call path.
deno_telemetry::init(
deno_telemetry::OtelRuntimeConfig {
runtime_name: "windmill-nativets".into(),
runtime_version: "0".into(),
},
deno_telemetry::OtelConfig {
tracing_enabled: true,
console: deno_telemetry::OtelConsoleConfig::Capture,
..Default::default()
},
)
.expect("init with tracing_enabled config must succeed on a fresh OnceCell");
assert!(
deno_telemetry::OTEL_GLOBALS.get().is_some(),
"load_internal_otel_exporter's OtelConfig must populate \
OTEL_GLOBALS so OtelTracer::builtin() does not panic on \
.unwrap() once __bootstrapOtel flips JS-side TRACING_ENABLED"
);
}
+1 -5
View File
@@ -39,7 +39,7 @@ ruby = ["dep:windmill-parser-ruby"]
rlang = ["dep:windmill-parser-r"]
duckdb = ["dep:libloading"]
quickjs = ["windmill-jseval/quickjs"]
bedrock = ["windmill-ai/bedrock", "dep:aws-sdk-bedrockruntime", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"]
bedrock = ["windmill-ai/bedrock"]
[dependencies]
windmill-ai = { workspace = true, default-features = false }
@@ -71,10 +71,6 @@ windmill-parser-sql.workspace = true
windmill-parser-graphql.workspace = true
windmill-parser-php = { workspace = true, optional = true }
windmill-git-sync.workspace = true
aws-sdk-bedrockruntime = { workspace = true, optional = true }
aws-config = { workspace = true, optional = true }
aws-credential-types = { workspace = true, optional = true }
aws-smithy-types = { workspace = true, optional = true }
flume.workspace = true
sqlx.workspace = true
uuid.workspace = true
-1
View File
@@ -1,7 +1,6 @@
// AI executor module structure
// This module will contain all AI-related execution logic
pub mod providers;
pub mod query_builder;
pub mod tools;
pub mod utils;

Some files were not shown because too many files have changed in this diff Show More