mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-20 00:02:19 +00:00
merge
This commit is contained in:
@@ -20,6 +20,8 @@ on:
|
||||
type: string
|
||||
default: ''
|
||||
secrets:
|
||||
OPENAI_API_KEY:
|
||||
required: false
|
||||
CODEX_AUTH_JSON:
|
||||
required: false
|
||||
WINDMILL_EE_PRIVATE_ACCESS:
|
||||
@@ -60,13 +62,18 @@ jobs:
|
||||
- name: Check Codex configuration
|
||||
id: codex_config
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
|
||||
run: |
|
||||
if [ -n "$CODEX_AUTH_JSON" ]; then
|
||||
if [ -n "$OPENAI_API_KEY" ]; then
|
||||
echo "enabled=true" >> "$GITHUB_OUTPUT"
|
||||
echo "auth_mode=api_key" >> "$GITHUB_OUTPUT"
|
||||
elif [ -n "$CODEX_AUTH_JSON" ]; then
|
||||
echo "enabled=true" >> "$GITHUB_OUTPUT"
|
||||
echo "auth_mode=oauth_json" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "enabled=false" >> "$GITHUB_OUTPUT"
|
||||
echo "CODEX_AUTH_JSON is not configured; skipping Codex review."
|
||||
echo "Codex auth is not configured; set OPENAI_API_KEY or CODEX_AUTH_JSON to enable Codex review."
|
||||
fi
|
||||
|
||||
- name: Resolve PR metadata
|
||||
@@ -169,9 +176,10 @@ jobs:
|
||||
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
|
||||
run: npm install --global @openai/codex@0.128.0
|
||||
|
||||
- name: Configure file-backed Codex auth
|
||||
- name: Configure Codex auth
|
||||
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
|
||||
run: |
|
||||
CODEX_HOME="$HOME/.codex"
|
||||
@@ -181,9 +189,13 @@ jobs:
|
||||
cat > "$CODEX_HOME/config.toml" <<'EOF'
|
||||
cli_auth_credentials_store = "file"
|
||||
EOF
|
||||
printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json"
|
||||
chmod 600 "$CODEX_HOME/auth.json"
|
||||
node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json"
|
||||
if [ -n "$OPENAI_API_KEY" ]; then
|
||||
printf '%s' "$OPENAI_API_KEY" | codex login --with-api-key
|
||||
else
|
||||
printf '%s' "$CODEX_AUTH_JSON" > "$CODEX_HOME/auth.json"
|
||||
chmod 600 "$CODEX_HOME/auth.json"
|
||||
node -e 'JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"))' "$CODEX_HOME/auth.json"
|
||||
fi
|
||||
|
||||
- name: Pre-fetch base and head refs for the PR
|
||||
if: steps.codex_config.outputs.enabled == 'true' && steps.pr.outputs.skip != 'true'
|
||||
|
||||
@@ -100,6 +100,7 @@ jobs:
|
||||
extra_prompt: ${{ needs.parse.outputs.extra_prompt }}
|
||||
triggered_by: ${{ github.event.comment.user.login }}
|
||||
secrets:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
CODEX_AUTH_JSON: ${{ secrets.CODEX_AUTH_JSON }}
|
||||
WINDMILL_EE_PRIVATE_ACCESS: ${{ secrets.WINDMILL_EE_PRIVATE_ACCESS }}
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
name: Publish CLI docs repo
|
||||
|
||||
# Regenerates the windmill-cli-docs repo (consumed by context7) from the
|
||||
# canonical sources in this repo on every Windmill release.
|
||||
#
|
||||
# Required secret:
|
||||
# CLI_DOCS_DEPLOY_KEY — ed25519 private key whose public half is registered
|
||||
# as a write-access deploy key on
|
||||
# windmill-labs/windmill-cli-docs.
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- "v*"
|
||||
workflow_dispatch:
|
||||
|
||||
# Serialize pushes to windmill-cli-docs so two release tags landing close
|
||||
# together (e.g. a release-please bump + a hotfix) can't race to force-push
|
||||
# the docs repo.
|
||||
concurrency:
|
||||
group: publish-cli-docs
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout windmill (source of truth)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
path: windmill
|
||||
|
||||
- name: Checkout windmill-cli-docs (publish target)
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
repository: windmill-labs/windmill-cli-docs
|
||||
path: windmill-cli-docs
|
||||
ssh-key: ${{ secrets.CLI_DOCS_DEPLOY_KEY }}
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pip install pyyaml
|
||||
|
||||
- name: Regenerate docs
|
||||
run: |
|
||||
python3 windmill/system_prompts/generate.py \
|
||||
--context7-dir "$GITHUB_WORKSPACE/windmill-cli-docs"
|
||||
|
||||
- name: Commit and push if changed
|
||||
working-directory: windmill-cli-docs
|
||||
env:
|
||||
REF_NAME: ${{ github.ref_name }}
|
||||
REF_TYPE: ${{ github.ref_type }}
|
||||
run: |
|
||||
git config user.name "windmill-bot"
|
||||
git config user.email "bot@windmill.dev"
|
||||
git add -A
|
||||
if git diff --cached --quiet; then
|
||||
echo "No doc changes for ${REF_NAME}."
|
||||
committed=false
|
||||
else
|
||||
committed=true
|
||||
if [ "${REF_TYPE}" = "tag" ]; then
|
||||
git commit -m "chore: sync from windmill ${REF_NAME}"
|
||||
else
|
||||
git commit -m "chore: sync from windmill (manual dispatch from ${REF_NAME})"
|
||||
fi
|
||||
git push origin HEAD
|
||||
fi
|
||||
# Always mirror the version tag on tag pushes, even when content
|
||||
# didn't change — downstream consumers tie snapshots to releases by
|
||||
# tag, and skipping it would leave the docs repo without a tag for
|
||||
# the new Windmill release.
|
||||
# workflow_dispatch from a non-tag ref skips this so we don't
|
||||
# create a junk tag named after a branch.
|
||||
if [ "${REF_TYPE}" = "tag" ]; then
|
||||
git tag -f "${REF_NAME}"
|
||||
git push origin "${REF_NAME}" --force
|
||||
echo "Mirrored tag ${REF_NAME} to windmill-cli-docs (content changed: ${committed})."
|
||||
fi
|
||||
@@ -20,6 +20,7 @@ rust-client/Cargo.toml
|
||||
|
||||
# Worktree-specific Claude Code settings (generated by scripts/worktree-env)
|
||||
.claude/settings.local.json
|
||||
.claude/worktrees/
|
||||
|
||||
# Symlinked cache directories (for git worktrees)
|
||||
backend/target
|
||||
|
||||
@@ -1,5 +1,88 @@
|
||||
# Changelog
|
||||
|
||||
## [1.702.1](https://github.com/windmill-labs/windmill/compare/v1.702.0...v1.702.1) (2026-05-14)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **nativets:** pass tracing-enabled OtelConfig to deno_telemetry::init ([#9163](https://github.com/windmill-labs/windmill/issues/9163)) ([bf99283](https://github.com/windmill-labs/windmill/commit/bf99283c3333bcdbc7679f4aea04ba29e41a48a5))
|
||||
|
||||
## [1.702.0](https://github.com/windmill-labs/windmill/compare/v1.701.0...v1.702.0) (2026-05-14)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **git-sync:** sync extra_perms for flows/scripts/apps ([#9162](https://github.com/windmill-labs/windmill/issues/9162)) ([5e909b2](https://github.com/windmill-labs/windmill/commit/5e909b2b4f2819f19deaf06d9e78e6458b324683))
|
||||
* include service accounts in instance settings users list ([#9157](https://github.com/windmill-labs/windmill/issues/9157)) ([e5286f4](https://github.com/windmill-labs/windmill/commit/e5286f46074cf2893e6ccd26175f929f16011c8f))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **mcp:** sanitize and enrich nested resource schemas ([#9158](https://github.com/windmill-labs/windmill/issues/9158)) ([d870edc](https://github.com/windmill-labs/windmill/commit/d870edc959481a06c894b4eda5e2be1a0269d7d0))
|
||||
|
||||
## [1.701.0](https://github.com/windmill-labs/windmill/compare/v1.700.2...v1.701.0) (2026-05-13)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **frontend:** unified EditorHeader with file picker for flow/script/app editors ([#9047](https://github.com/windmill-labs/windmill/issues/9047)) ([d0f23cc](https://github.com/windmill-labs/windmill/commit/d0f23cc5238b025208c61e983701894de28536d5))
|
||||
* read-only flag on API tokens ([#9144](https://github.com/windmill-labs/windmill/issues/9144)) ([d666e84](https://github.com/windmill-labs/windmill/commit/d666e8431cdbf14d9373d9ef625b5aafc50ac50a))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* align script path existence check with deploy logic; hide Delete for non-admin ([#9152](https://github.com/windmill-labs/windmill/issues/9152)) ([c509206](https://github.com/windmill-labs/windmill/commit/c5092069cbeda2c4c18bea80dd629c7c087b30bf))
|
||||
* Allow devops role to use all_workspaces runs filter in admins workspace ([#9153](https://github.com/windmill-labs/windmill/issues/9153)) ([110bef0](https://github.com/windmill-labs/windmill/commit/110bef0a6e76615c7b371c5c0f5bc1f4e7a73a64))
|
||||
* **bun:** pass --preserve-symlinks on unbundled execution ([#9147](https://github.com/windmill-labs/windmill/issues/9147)) ([4d0f2c2](https://github.com/windmill-labs/windmill/commit/4d0f2c26a116a0f8a89a64231dc824eabda0a8c3))
|
||||
* **cli:** prevent !inline-corruption in flow push/pull ([#9142](https://github.com/windmill-labs/windmill/issues/9142)) ([79c5b7b](https://github.com/windmill-labs/windmill/commit/79c5b7b8b7676b0a06fa6480dd04b7105d39d250))
|
||||
* **operator:** refresh IAM RDS / Entra ID tokens in operator process ([#9141](https://github.com/windmill-labs/windmill/issues/9141)) ([7ebb081](https://github.com/windmill-labs/windmill/commit/7ebb08133cd4027bc00bacc4a0fc5865cd5709ec))
|
||||
* **python:** preserve strings containing Infinity/NaN in result JSON ([#9149](https://github.com/windmill-labs/windmill/issues/9149)) ([33bf01b](https://github.com/windmill-labs/windmill/commit/33bf01b627c8ea430c03dfc27a97a8f2d770582f))
|
||||
* scope promotion-mode debounce key per repo ([#9145](https://github.com/windmill-labs/windmill/issues/9145)) ([2ec1863](https://github.com/windmill-labs/windmill/commit/2ec1863340e759bba3408dbc4f41b16912b959ea))
|
||||
* send flow push-loop ping outside transaction so zombie monitor sees it ([#9136](https://github.com/windmill-labs/windmill/issues/9136)) ([818cb31](https://github.com/windmill-labs/windmill/commit/818cb31fbc731fa5c70ddf5942bb37bc4bc56e4d))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* **dynselect:** only retrigger when helper args actually change ([#9148](https://github.com/windmill-labs/windmill/issues/9148)) ([dd19e52](https://github.com/windmill-labs/windmill/commit/dd19e52a84fb9a9f48e3ad061b084841c2ee7464))
|
||||
|
||||
## [1.700.2](https://github.com/windmill-labs/windmill/compare/v1.700.1...v1.700.2) (2026-05-12)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* preserve explicit nulls for typed fields in bulk instance config ([#9123](https://github.com/windmill-labs/windmill/issues/9123)) ([cab0000](https://github.com/windmill-labs/windmill/commit/cab0000f3a5e9a0b201a85da1a01b1f82df8a316))
|
||||
* preserve negative integers in Bedrock tool schema conversion ([#9116](https://github.com/windmill-labs/windmill/issues/9116)) ([01e21c7](https://github.com/windmill-labs/windmill/commit/01e21c7f913eaf7dffc3d6a31501418ff2104c8b))
|
||||
|
||||
## [1.700.1](https://github.com/windmill-labs/windmill/compare/v1.700.0...v1.700.1) (2026-05-11)
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* CE build broken by enterprise-gated compute_instance_hash ([#9113](https://github.com/windmill-labs/windmill/issues/9113)) ([cd65de4](https://github.com/windmill-labs/windmill/commit/cd65de49285ff60abdd94c883180ded65609f382))
|
||||
|
||||
## [1.700.0](https://github.com/windmill-labs/windmill/compare/v1.699.0...v1.700.0) (2026-05-11)
|
||||
|
||||
|
||||
### Features
|
||||
|
||||
* **cli:** auto-infer args for `wmill app push` ([#9091](https://github.com/windmill-labs/windmill/issues/9091)) ([43b1800](https://github.com/windmill-labs/windmill/commit/43b18006f32fd5db54bbf8ae7ff0e0b314a517e5))
|
||||
* **forks:** prompt to delete forked children when deleting a fork ([#9097](https://github.com/windmill-labs/windmill/issues/9097)) ([e43a958](https://github.com/windmill-labs/windmill/commit/e43a958c5c6ae01a1fbecf3db63c6541a245be62))
|
||||
* **operators:** allow operators to access assets page ([#9095](https://github.com/windmill-labs/windmill/issues/9095)) ([20ecd90](https://github.com/windmill-labs/windmill/commit/20ecd904e7060c3cf90f2605740bb349b2a3e6ed))
|
||||
* **vault:** configurable JWT auth mount path and setup-doc fixes ([#9100](https://github.com/windmill-labs/windmill/issues/9100)) ([f8ba084](https://github.com/windmill-labs/windmill/commit/f8ba0840d74572c880cf458938365b3ec808c6fb))
|
||||
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* add Input, Result, Trigger to reserved flow step IDs ([#9109](https://github.com/windmill-labs/windmill/issues/9109)) ([9f79a86](https://github.com/windmill-labs/windmill/commit/9f79a86a686708f66ccc512d4f132cb9a00397a7)), closes [#7139](https://github.com/windmill-labs/windmill/issues/7139)
|
||||
* **frontend:** mark Path dirty when folder picker changes selection ([#9096](https://github.com/windmill-labs/windmill/issues/9096)) ([23bb1b5](https://github.com/windmill-labs/windmill/commit/23bb1b541e78846d5978153fd8d9bb4f01cec72b))
|
||||
* mask oauth client secret in instance settings ([#9112](https://github.com/windmill-labs/windmill/issues/9112)) ([ac3c155](https://github.com/windmill-labs/windmill/commit/ac3c155541eb5ca20d65c38ad13dca6c10a572c9))
|
||||
* populate raw_code for flowscript and appscript runs ([#9104](https://github.com/windmill-labs/windmill/issues/9104)) ([05172ac](https://github.com/windmill-labs/windmill/commit/05172ac3bdfc3472da5e9d8a825cdd479ba9e375))
|
||||
|
||||
|
||||
### Performance Improvements
|
||||
|
||||
* lazy-load script editor history and hit partial index ([#9107](https://github.com/windmill-labs/windmill/issues/9107)) ([03e8bc8](https://github.com/windmill-labs/windmill/commit/03e8bc8c14258355d7d695333c1588807fbf8cd6))
|
||||
|
||||
## [1.699.0](https://github.com/windmill-labs/windmill/compare/v1.698.0...v1.699.0) (2026-05-08)
|
||||
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ This folder contains black-box benchmark cases for:
|
||||
- `app`
|
||||
- `script`
|
||||
- `cli`
|
||||
- `global`
|
||||
|
||||
The goal is to test the current production prompts and guidance with realistic user requests, not to test one exact implementation shape.
|
||||
|
||||
@@ -75,6 +76,16 @@ Still, avoid benchmark phrasing. The prompt should read like a repo task, not a
|
||||
|
||||
When relevant, ask the assistant to tell the user which `wmill` commands to run next. That is part of the benchmarked behavior.
|
||||
|
||||
## Global-specific rules
|
||||
|
||||
Global prompts should exercise workspace-level drafting behavior:
|
||||
|
||||
- inspecting existing scripts, flows, apps, schedules, triggers, resources, and variables when relevant
|
||||
- writing AI drafts rather than saving or deploying by default
|
||||
- producing coherent multi-artifact changes when the request crosses artifact boundaries
|
||||
|
||||
Keep deterministic validation focused on the draft contract: required draft type/path, required content snippets, forbidden draft paths, and forbidden mutating tools such as deploy/delete unless the case explicitly asks for them.
|
||||
|
||||
## Deterministic validation
|
||||
|
||||
Use deterministic validation only for hard failures such as:
|
||||
|
||||
+21
-13
@@ -1,11 +1,12 @@
|
||||
# AI Evals
|
||||
|
||||
Small benchmark runner for the four Windmill AI generation modes:
|
||||
Small benchmark runner for the Windmill AI generation modes:
|
||||
|
||||
- `cli`
|
||||
- `flow`
|
||||
- `script`
|
||||
- `app`
|
||||
- `global`
|
||||
|
||||
The benchmark always tests the current production prompts, tools, and guidance in this checkout.
|
||||
|
||||
@@ -55,8 +56,9 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus
|
||||
bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o
|
||||
bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose
|
||||
bun run cli -- run flow --record
|
||||
GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro --transport proxy
|
||||
GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro
|
||||
WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview
|
||||
bun run cli -- run global global-test1-script-create
|
||||
bun run cli -- run cli bun-hello-script
|
||||
```
|
||||
|
||||
@@ -72,7 +74,6 @@ Public CLI surface:
|
||||
- `--output <path>`: custom result JSON path
|
||||
- `--model <alias>`: choose the model under test
|
||||
- `--models <a,b,c>`: run the same cases sequentially against several model aliases
|
||||
- `--transport <mode>`: frontend request transport (`direct` by default, `proxy` to exercise `/api/w/{workspace}/ai/proxy`)
|
||||
- `--verbose`: stream assistant output for frontend runs
|
||||
- `--record`: append a compact tracked summary line to `ai_evals/history/<mode>.jsonl` for full-suite runs only
|
||||
- `--backend-validation <mode>`: optional backend smoke validation (`off` or `preview`) for `script` and `flow` evals
|
||||
@@ -95,7 +96,7 @@ Today:
|
||||
Notes:
|
||||
|
||||
- the command also prints accepted alias spellings such as `gpt-4o`, `claude-opus-4.6`, and `claude-haiku-4.5`
|
||||
- frontend modes (`flow`, `script`, `app`) can use Anthropic, OpenAI, and Gemini-backed aliases
|
||||
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, and Gemini-backed aliases
|
||||
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
|
||||
- the judge model is separate and currently defaults to `claude-sonnet-4-6`
|
||||
|
||||
@@ -134,6 +135,13 @@ For `app` mode, `validate` can express narrow hard requirements such as:
|
||||
- minimum datatable / datatable-table counts
|
||||
- specific required datatable tables
|
||||
|
||||
For `global` mode, `validate` can express draft-level requirements such as:
|
||||
|
||||
- required draft type/path/language
|
||||
- required or forbidden snippets in draft values
|
||||
- required or forbidden draft counts
|
||||
- forbidden draft paths
|
||||
|
||||
App fixtures can also include an optional `datatables.json` file at the fixture root.
|
||||
|
||||
For `flow` mode, an `initial` fixture can also include a benchmark workspace catalog of
|
||||
@@ -145,26 +153,23 @@ If `--backend-validation preview` is enabled:
|
||||
- `script` evals run a real backend script preview in an isolated temp workspace
|
||||
- `flow` evals run a real backend flow preview only for cases that define `runtime.backendPreview`
|
||||
- `flow` cases with `initial.workspace` fixtures seed those scripts and flows into the preview workspace before preview
|
||||
- when `WMILL_AI_EVAL_BACKEND_WORKSPACE` is set, `ai_evals` treats that workspace as a dedicated test workspace, clears managed eval assets under `f/evals/*` before each preview run, and then reseeds the current case fixtures
|
||||
- when `WMILL_AI_EVAL_BACKEND_WORKSPACE` is set, `ai_evals` creates or reuses that workspace as a dedicated test workspace, clears managed eval assets under `f/evals/*` before each preview run, and then reseeds the current case fixtures
|
||||
|
||||
Supported backend validation env vars:
|
||||
Supported backend env vars:
|
||||
|
||||
- `WMILL_AI_EVAL_BACKEND_VALIDATION=preview`
|
||||
- `WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000`
|
||||
- `WMILL_AI_EVAL_BACKEND_EMAIL=admin@windmill.dev`
|
||||
- `WMILL_AI_EVAL_BACKEND_PASSWORD=changeme`
|
||||
- `WMILL_AI_EVAL_BACKEND_WORKSPACE=integration-tests` to reuse an existing workspace on CE installs with low workspace limits
|
||||
- `WMILL_AI_EVAL_KEEP_WORKSPACES=1`
|
||||
- `WMILL_AI_EVAL_WORKSPACE_PREFIX=ai-evals`
|
||||
|
||||
Frontend proxy transport uses the same backend auth/workspace env vars.
|
||||
Frontend modes require a reachable Windmill backend and send model requests through the workspace AI proxy at `/api/w/{workspace}/ai/proxy`. At startup, `ai_evals` checks the resolved backend URL and fails early with setup guidance if the backend cannot be reached or login fails.
|
||||
|
||||
When `--transport proxy` is set:
|
||||
For frontend modes:
|
||||
|
||||
- `ai_evals` creates or reuses a backend workspace
|
||||
- `ai_evals` creates a temporary backend workspace, or creates/reuses `WMILL_AI_EVAL_BACKEND_WORKSPACE` when it is set
|
||||
- it upserts a provider resource under `f/evals/ai/<provider>`
|
||||
- frontend requests go through `/api/w/{workspace}/ai/proxy`
|
||||
- result JSON and history records include `transport` so direct vs proxy runs stay distinguishable
|
||||
|
||||
## Results And Artifacts
|
||||
|
||||
@@ -178,11 +183,12 @@ If `--record` is used, the CLI also appends one compact JSON line to:
|
||||
- `ai_evals/history/flow.jsonl`
|
||||
- `ai_evals/history/script.jsonl`
|
||||
- `ai_evals/history/app.jsonl`
|
||||
- `ai_evals/history/global.jsonl`
|
||||
- `ai_evals/history/cli.jsonl`
|
||||
|
||||
Each recorded line contains:
|
||||
|
||||
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `transport`, `judgeModel`)
|
||||
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`)
|
||||
- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`)
|
||||
- average token usage (`averageTokenUsagePerAttempt`)
|
||||
- per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate)
|
||||
@@ -198,6 +204,7 @@ Typical artifacts by mode:
|
||||
- `flow`: `flow.json`
|
||||
- `script`: `script.json` plus the generated script file
|
||||
- `app`: `app.json` plus frontend/backend files
|
||||
- `global`: `global-drafts.json`
|
||||
- `cli`: `assistant-output.txt`, `trace.json`, `wmill-invocations.jsonl`, plus generated workspace files
|
||||
- backend-validated attempts also include `backend-preview.json`
|
||||
|
||||
@@ -213,6 +220,7 @@ Typical artifacts by mode:
|
||||
## Notes
|
||||
|
||||
- Frontend modes reuse the production frontend chat code through the Vitest bridge.
|
||||
- Global mode evaluates the production global AI tools and validates the resulting AI draft store.
|
||||
- CLI mode creates an isolated workspace, writes the current checkout guidance into it, and benchmarks the real skills / `AGENTS.md` flow.
|
||||
- CLI mode now also records a structured trace of invoked skills, tool calls, proposed `wmill` commands, and any attempted `wmill` executions.
|
||||
- Frontend progress streams live while the benchmark is running.
|
||||
|
||||
@@ -210,8 +210,6 @@ function buildSettings(
|
||||
baseUrl: 'http://backend.test/default',
|
||||
email: 'admin@windmill.dev',
|
||||
password: 'changeme',
|
||||
keepWorkspaces: true,
|
||||
workspacePrefix: 'ai-evals',
|
||||
pollIntervalMs: 1,
|
||||
maxWaitMs: 50,
|
||||
...overrides
|
||||
|
||||
@@ -24,6 +24,7 @@ export interface CompletedPreviewJob {
|
||||
const tokenCache = new Map<string, Promise<string>>()
|
||||
const sharedWorkspaceQueue = new Map<string, Promise<void>>()
|
||||
const managedSharedWorkspacePrefixes = ['f/evals/']
|
||||
const DEFAULT_WORKSPACE_PREFIX = 'ai-evals'
|
||||
|
||||
export class BackendPreviewClient {
|
||||
constructor(private readonly settings: BackendValidationSettings) {}
|
||||
@@ -35,7 +36,7 @@ export class BackendPreviewClient {
|
||||
): Promise<T> {
|
||||
const workspaceId =
|
||||
this.settings.workspaceOverride ??
|
||||
buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt)
|
||||
buildWorkspaceId(caseId, attempt)
|
||||
|
||||
const run = async () => {
|
||||
await this.ensureWorkspace(workspaceId)
|
||||
@@ -46,7 +47,7 @@ export class BackendPreviewClient {
|
||||
try {
|
||||
return await body(workspaceId)
|
||||
} finally {
|
||||
if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) {
|
||||
if (!this.settings.workspaceOverride) {
|
||||
await this.deleteWorkspace(workspaceId).catch(() => undefined)
|
||||
}
|
||||
}
|
||||
@@ -440,14 +441,14 @@ async function withSharedWorkspaceLock<T>(workspaceId: string, body: () => Promi
|
||||
}
|
||||
}
|
||||
|
||||
function buildWorkspaceId(prefix: string, caseId: string, attempt: number): string {
|
||||
function buildWorkspaceId(caseId: string, attempt: number): string {
|
||||
const caseSlug = caseId
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.slice(0, 30)
|
||||
const suffix = randomUUID().slice(0, 8)
|
||||
return `${prefix}-${caseSlug || 'case'}-a${attempt}-${suffix}`
|
||||
return `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || 'case'}-a${attempt}-${suffix}`
|
||||
}
|
||||
|
||||
function extractFolderName(path: string): string | null {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { loadSelectedCases } from "../../core/cases";
|
||||
import { resolveBackendValidationSettings } from "../../core/backendValidation";
|
||||
import { resolveFrontendEvalTransportSettings } from "../../core/frontendTransport";
|
||||
import {
|
||||
formatRunModelLabel,
|
||||
getFrontendEvalModel,
|
||||
@@ -9,13 +8,15 @@ import {
|
||||
import { buildRunResult } from "../../core/results";
|
||||
import { runSuite } from "../../core/runSuite";
|
||||
import type { BenchmarkRunResult, ModeRunner } from "../../core/types";
|
||||
import { resolveWindmillBackendSettings } from "../../core/windmillBackendSettings";
|
||||
import { emitFrontendBenchmarkProgress } from "./progress";
|
||||
import { createAppModeRunner } from "../../modes/app";
|
||||
import { createFlowModeRunner } from "../../modes/flow";
|
||||
import { createGlobalModeRunner } from "../../modes/global";
|
||||
import { createScriptModeRunner } from "../../modes/script";
|
||||
import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
|
||||
|
||||
export type FrontendBenchmarkMode = "flow" | "app" | "script";
|
||||
export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global";
|
||||
|
||||
export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult> {
|
||||
const mode = parseMode(process.env.WMILL_FRONTEND_AI_EVAL_MODE);
|
||||
@@ -36,17 +37,14 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
|
||||
evalMode: mode,
|
||||
requestedMode: process.env.WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION,
|
||||
});
|
||||
const transportSettings = resolveFrontendEvalTransportSettings({
|
||||
evalMode: mode,
|
||||
requestedTransport: process.env.WMILL_FRONTEND_AI_EVAL_TRANSPORT,
|
||||
});
|
||||
const backendSettings = resolveWindmillBackendSettings();
|
||||
|
||||
const selectedCases = await loadSelectedCases(mode, caseIds);
|
||||
const modeRunner = getModeRunner(
|
||||
mode,
|
||||
getFrontendEvalModel(model),
|
||||
backendValidation,
|
||||
transportSettings,
|
||||
backendSettings,
|
||||
);
|
||||
const runModel = formatRunModelLabel(mode, model);
|
||||
const caseResults = await runSuite({
|
||||
@@ -66,7 +64,6 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
|
||||
mode,
|
||||
runs,
|
||||
runModel,
|
||||
transport: transportSettings.transport,
|
||||
judgeModel: DEFAULT_JUDGE_MODEL,
|
||||
caseResults,
|
||||
});
|
||||
@@ -76,24 +73,26 @@ function getModeRunner(
|
||||
mode: FrontendBenchmarkMode,
|
||||
model: ReturnType<typeof getFrontendEvalModel>,
|
||||
backendValidation: ReturnType<typeof resolveBackendValidationSettings>,
|
||||
transportSettings: ReturnType<typeof resolveFrontendEvalTransportSettings>,
|
||||
backendSettings: ReturnType<typeof resolveWindmillBackendSettings>,
|
||||
): ModeRunner<any, any, any> {
|
||||
switch (mode) {
|
||||
case "flow":
|
||||
return createFlowModeRunner(model, backendValidation, transportSettings);
|
||||
return createFlowModeRunner(model, backendValidation, backendSettings);
|
||||
case "app":
|
||||
return createAppModeRunner(model, transportSettings);
|
||||
return createAppModeRunner(model, backendSettings);
|
||||
case "script":
|
||||
return createScriptModeRunner(
|
||||
model,
|
||||
backendValidation,
|
||||
transportSettings,
|
||||
backendSettings,
|
||||
);
|
||||
case "global":
|
||||
return createGlobalModeRunner(model, backendSettings);
|
||||
}
|
||||
}
|
||||
|
||||
function parseMode(value: string | undefined): FrontendBenchmarkMode {
|
||||
if (value === "flow" || value === "app" || value === "script") {
|
||||
if (value === "flow" || value === "app" || value === "script" || value === "global") {
|
||||
return value;
|
||||
}
|
||||
throw new Error(`Unsupported frontend benchmark mode: ${String(value)}`);
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
prepareAppUserMessage,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/app/core";
|
||||
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
|
||||
import { createAppFileHelpers } from "./fileHelpers";
|
||||
import { createAppFileHelpers, type AppEvalChatHelpers } from "./fileHelpers";
|
||||
import { runEval } from "../shared";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import type {
|
||||
@@ -22,7 +22,6 @@ import type {
|
||||
} from "../../../../core/types";
|
||||
import type { TokenUsage } from "../shared/types";
|
||||
import type { AppFilesState } from "../../../../core/validators";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
import {
|
||||
createAppBackendRunnableContextElement,
|
||||
@@ -49,8 +48,7 @@ export interface AppEvalOptions {
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
backend: WindmillBackendSettings;
|
||||
workspaceRoot?: string;
|
||||
runContext?: ModeRunContext;
|
||||
}
|
||||
@@ -58,7 +56,7 @@ export interface AppEvalOptions {
|
||||
export async function runAppEval(
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options?: AppEvalOptions,
|
||||
options: AppEvalOptions,
|
||||
): Promise<AppEvalResult> {
|
||||
const workspaceRoot =
|
||||
options?.workspaceRoot ??
|
||||
@@ -101,10 +99,9 @@ export async function runAppEval(
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: options?.provider,
|
||||
transport: options?.transport,
|
||||
backend: options?.backend,
|
||||
proxyCaseId: options?.runContext?.caseId,
|
||||
proxyAttempt: options?.runContext?.attempt,
|
||||
backend: options.backend,
|
||||
caseId: options?.runContext?.caseId,
|
||||
attempt: options?.runContext?.attempt,
|
||||
},
|
||||
});
|
||||
|
||||
@@ -124,7 +121,7 @@ export async function runAppEval(
|
||||
|
||||
async function buildAdditionalContext(
|
||||
appContext: EvalCaseRuntimeAppContextSpec | undefined,
|
||||
helpers: AppAIChatHelpers,
|
||||
helpers: AppEvalChatHelpers,
|
||||
): Promise<ContextElement[]> {
|
||||
const entries = appContext?.additional ?? [];
|
||||
if (entries.length === 0) {
|
||||
|
||||
@@ -2,6 +2,7 @@ import { mkdir, rm, writeFile } from 'fs/promises'
|
||||
import { dirname, join } from 'path'
|
||||
import type {
|
||||
AppAIChatHelpers,
|
||||
AppDatatableMetadata,
|
||||
AppFiles,
|
||||
BackendRunnable,
|
||||
DataTableSchema,
|
||||
@@ -10,6 +11,10 @@ import type {
|
||||
} from '../../../../../frontend/src/lib/components/copilot/chat/app/core'
|
||||
import { buildAppWmillTypes, collectAppDiagnostics } from '../../../../core/appDiagnostics'
|
||||
|
||||
export interface AppEvalChatHelpers extends AppAIChatHelpers {
|
||||
getDatatables: () => Promise<DataTableSchema[]>
|
||||
}
|
||||
|
||||
async function writeFrontendFile(
|
||||
workspaceRoot: string | undefined,
|
||||
path: string,
|
||||
@@ -92,7 +97,7 @@ export async function createAppFileHelpers(
|
||||
initialDatatables: DataTableSchema[] = [],
|
||||
workspaceRoot?: string
|
||||
): Promise<{
|
||||
helpers: AppAIChatHelpers
|
||||
helpers: AppEvalChatHelpers
|
||||
getFiles: () => AppFiles
|
||||
getEvalState: () => {
|
||||
frontend: Record<string, string>
|
||||
@@ -137,7 +142,7 @@ export async function createAppFileHelpers(
|
||||
}
|
||||
await persistDatatables(workspaceRoot, datatables)
|
||||
|
||||
const helpers: AppAIChatHelpers = {
|
||||
const helpers: AppEvalChatHelpers = {
|
||||
listFrontendFiles: () => [
|
||||
...Object.keys(frontend).filter((path) => path !== '/wmill.d.ts'),
|
||||
'/wmill.d.ts'
|
||||
@@ -211,6 +216,34 @@ export async function createAppFileHelpers(
|
||||
},
|
||||
lint,
|
||||
getDatatables: async () => structuredClone(datatables),
|
||||
listDatatableTables: async () =>
|
||||
datatables.map(
|
||||
(datatable): AppDatatableMetadata => {
|
||||
const schemas = Object.fromEntries(
|
||||
Object.entries(datatable.schemas).map(([schemaName, tables]) => [
|
||||
schemaName,
|
||||
Object.keys(tables)
|
||||
])
|
||||
)
|
||||
return {
|
||||
datatable_name: datatable.datatable_name,
|
||||
schemas,
|
||||
tableCount: Object.values(schemas).reduce(
|
||||
(sum, tableNames) => sum + tableNames.length,
|
||||
0
|
||||
),
|
||||
error: datatable.error
|
||||
}
|
||||
}
|
||||
),
|
||||
getDatatableTableSchema: async (
|
||||
datatableName: string,
|
||||
schemaName: string,
|
||||
tableName: string
|
||||
) => {
|
||||
const datatable = datatables.find((entry) => entry.datatable_name === datatableName)
|
||||
return structuredClone(datatable?.schemas?.[schemaName]?.[tableName] ?? {})
|
||||
},
|
||||
getAvailableDatatableNames: () => datatables.map((datatable) => datatable.datatable_name),
|
||||
execDatatableSql: async (
|
||||
datatableName: string,
|
||||
|
||||
@@ -18,7 +18,6 @@ import {
|
||||
import { runEval } from "../shared";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { TokenUsage, ToolCallDetail } from "../shared/types";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
|
||||
export interface FlowFixture {
|
||||
@@ -48,8 +47,7 @@ export interface FlowEvalOptions {
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
backend: WindmillBackendSettings;
|
||||
workspaceRoot?: string;
|
||||
runContext?: ModeRunContext;
|
||||
}
|
||||
@@ -57,7 +55,7 @@ export interface FlowEvalOptions {
|
||||
export async function runFlowEval(
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options?: FlowEvalOptions,
|
||||
options: FlowEvalOptions,
|
||||
): Promise<FlowEvalResult> {
|
||||
const workspaceRoot =
|
||||
options?.workspaceRoot ??
|
||||
@@ -100,10 +98,9 @@ export async function runFlowEval(
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: options?.provider,
|
||||
transport: options?.transport,
|
||||
backend: options?.backend,
|
||||
proxyCaseId: options?.runContext?.caseId,
|
||||
proxyAttempt: options?.runContext?.attempt,
|
||||
backend: options.backend,
|
||||
caseId: options?.runContext?.caseId,
|
||||
attempt: options?.runContext?.attempt,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
import { mkdtemp, rm } from "fs/promises";
|
||||
import { tmpdir } from "os";
|
||||
import { join } from "path";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import {
|
||||
globalTools,
|
||||
prepareGlobalSystemMessage,
|
||||
prepareGlobalUserMessage,
|
||||
} from "../../../../../frontend/src/lib/components/copilot/chat/global/core";
|
||||
import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte";
|
||||
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { GlobalDraftState } from "../../../../core/validators";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
import {
|
||||
registerBenchmarkWorkspaceRunnables,
|
||||
unregisterBenchmarkWorkspaceRunnables,
|
||||
type BenchmarkWorkspaceRunnables,
|
||||
} from "../../mockBackend";
|
||||
import { runEval } from "../shared";
|
||||
import type { TokenUsage, ToolCallDetail } from "../shared/types";
|
||||
|
||||
const MUTATING_GLOBAL_TOOLS = new Set([
|
||||
"deploy_workspace_item",
|
||||
"delete_workspace_item",
|
||||
]);
|
||||
|
||||
export interface GlobalEvalResult {
|
||||
success: boolean;
|
||||
state: GlobalDraftState;
|
||||
error?: string;
|
||||
assistantMessageCount: number;
|
||||
toolCallCount: number;
|
||||
toolsUsed: string[];
|
||||
toolCallDetails: ToolCallDetail[];
|
||||
tokenUsage: TokenUsage;
|
||||
}
|
||||
|
||||
export interface GlobalEvalOptions {
|
||||
workspaceFixtures?: BenchmarkWorkspaceRunnables;
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
backend: WindmillBackendSettings;
|
||||
workspaceRoot?: string;
|
||||
runContext?: ModeRunContext;
|
||||
}
|
||||
|
||||
export async function runGlobalEval(
|
||||
userPrompt: string,
|
||||
apiKey: string,
|
||||
options: GlobalEvalOptions,
|
||||
): Promise<GlobalEvalResult> {
|
||||
const workspaceRoot =
|
||||
options.workspaceRoot ??
|
||||
(await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-")));
|
||||
|
||||
globalDraftStore.clearDrafts(workspaceRoot);
|
||||
registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {});
|
||||
|
||||
try {
|
||||
const model = options.model ?? "claude-haiku-4-5-20251001";
|
||||
const rawResult = await runEval({
|
||||
userPrompt,
|
||||
systemMessage: prepareGlobalSystemMessage(),
|
||||
userMessage: prepareGlobalUserMessage(userPrompt),
|
||||
tools: getGlobalEvalTools(),
|
||||
helpers: {},
|
||||
apiKey,
|
||||
getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }),
|
||||
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
|
||||
onAssistantToken: options.runContext?.onAssistantChunk,
|
||||
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
|
||||
onToolCall: options.runContext?.onToolCall,
|
||||
options: {
|
||||
maxIterations: options.maxIterations,
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: options.provider,
|
||||
backend: options.backend,
|
||||
caseId: options.runContext?.caseId,
|
||||
attempt: options.runContext?.attempt,
|
||||
},
|
||||
});
|
||||
|
||||
return {
|
||||
state: rawResult.output,
|
||||
success: rawResult.success,
|
||||
error: rawResult.error,
|
||||
assistantMessageCount: rawResult.iterations,
|
||||
toolCallCount: rawResult.toolCallsCount,
|
||||
toolsUsed: rawResult.toolsCalled,
|
||||
toolCallDetails: rawResult.toolCallDetails,
|
||||
tokenUsage: rawResult.tokenUsage,
|
||||
};
|
||||
} finally {
|
||||
globalDraftStore.clearDrafts(workspaceRoot);
|
||||
unregisterBenchmarkWorkspaceRunnables(workspaceRoot);
|
||||
if (!options.workspaceRoot) {
|
||||
await rm(workspaceRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getGlobalEvalTools(): ProductionTool<{}>[] {
|
||||
return (globalTools as ProductionTool<{}>[]).map((tool) => {
|
||||
if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) {
|
||||
return tool;
|
||||
}
|
||||
|
||||
return {
|
||||
...tool,
|
||||
requiresConfirmation: false,
|
||||
validateBeforeConfirmation: undefined,
|
||||
fn: async () =>
|
||||
JSON.stringify(
|
||||
{
|
||||
success: false,
|
||||
error:
|
||||
"This mutating workspace tool is disabled during ai_evals global mode.",
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
};
|
||||
});
|
||||
}
|
||||
@@ -14,7 +14,6 @@ import { createScriptFileHelpers, type ScriptEvalState } from "./fileHelpers";
|
||||
import { runEval } from "../shared";
|
||||
import type { ModeRunContext } from "../../../../core/types";
|
||||
import type { TokenUsage, ToolCallDetail } from "../shared/types";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
|
||||
export interface ScriptEvalResult {
|
||||
@@ -33,8 +32,7 @@ export interface ScriptEvalOptions {
|
||||
model?: string;
|
||||
maxIterations?: number;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
backend: WindmillBackendSettings;
|
||||
workspaceRoot?: string;
|
||||
runContext?: ModeRunContext;
|
||||
}
|
||||
@@ -98,10 +96,9 @@ export async function runScriptEval(
|
||||
model,
|
||||
workspace: workspaceRoot,
|
||||
provider: modelProvider.provider,
|
||||
transport: options.transport,
|
||||
backend: options.backend,
|
||||
proxyCaseId: options.runContext?.caseId,
|
||||
proxyAttempt: options.runContext?.attempt,
|
||||
caseId: options.runContext?.caseId,
|
||||
attempt: options.runContext?.attempt,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -40,8 +40,8 @@ export interface RunEvalParams<THelpers, TOutput> {
|
||||
apiKey: string;
|
||||
/** Function to get the current output state */
|
||||
getOutput: () => TOutput;
|
||||
/** Optional configuration */
|
||||
options?: EvalRunnerOptions;
|
||||
/** Model and Windmill backend configuration */
|
||||
options: EvalRunnerOptions;
|
||||
onAssistantMessageStart?: () => void;
|
||||
onAssistantToken?: (token: string) => void;
|
||||
onAssistantMessageEnd?: () => void;
|
||||
@@ -70,10 +70,10 @@ export async function runEval<THelpers, TOutput>(
|
||||
} = params;
|
||||
let shouldEmitMessageStart = true;
|
||||
|
||||
const model = options?.model ?? "gpt-4o";
|
||||
const maxIterations = options?.maxIterations ?? 20;
|
||||
const workspace = options?.workspace ?? "test-workspace";
|
||||
const provider = toFrontendEvalProvider(options?.provider);
|
||||
const model = options.model ?? "gpt-4o";
|
||||
const maxIterations = options.maxIterations ?? 20;
|
||||
const workspace = options.workspace ?? "test-workspace";
|
||||
const provider = toFrontendEvalProvider(options.provider);
|
||||
|
||||
const modelProvider = resolveEvalModelProvider(model, provider);
|
||||
|
||||
@@ -203,45 +203,31 @@ export async function runEval<THelpers, TOutput>(
|
||||
}
|
||||
};
|
||||
|
||||
if (options?.transport === "proxy") {
|
||||
const backendSettings = options.backend;
|
||||
if (!backendSettings) {
|
||||
throw new Error("Missing backend settings for proxy transport");
|
||||
}
|
||||
|
||||
const backendClient = new WindmillBackendClient(backendSettings);
|
||||
return await backendClient.withWorkspace(
|
||||
options.proxyCaseId ?? "eval",
|
||||
options.proxyAttempt ?? 1,
|
||||
async (proxyWorkspaceId) => {
|
||||
const resourcePath = buildProxyResourcePath(modelProvider.provider);
|
||||
await backendClient.upsertResource({
|
||||
workspaceId: proxyWorkspaceId,
|
||||
path: resourcePath,
|
||||
resourceType: modelProvider.provider,
|
||||
value: { api_key: apiKey },
|
||||
});
|
||||
const token = await backendClient.getToken();
|
||||
const clients = createEvalClients({
|
||||
provider: modelProvider.provider,
|
||||
apiKey,
|
||||
transport: "proxy",
|
||||
proxy: {
|
||||
baseURL: `${backendSettings.baseUrl}/api/w/${encodeURIComponent(proxyWorkspaceId)}/ai/proxy`,
|
||||
bearerToken: token,
|
||||
resourcePath,
|
||||
},
|
||||
}) as unknown as ChatClients;
|
||||
return await executeChatLoop(clients);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const clients = createEvalClients({
|
||||
provider: modelProvider.provider,
|
||||
apiKey,
|
||||
}) as unknown as ChatClients;
|
||||
return await executeChatLoop(clients);
|
||||
const backendSettings = options.backend;
|
||||
const backendClient = new WindmillBackendClient(backendSettings);
|
||||
return await backendClient.withWorkspace(
|
||||
options.caseId ?? "eval",
|
||||
options.attempt ?? 1,
|
||||
async (proxyWorkspaceId) => {
|
||||
const resourcePath = buildProxyResourcePath(modelProvider.provider);
|
||||
await backendClient.upsertResource({
|
||||
workspaceId: proxyWorkspaceId,
|
||||
path: resourcePath,
|
||||
resourceType: modelProvider.provider,
|
||||
value: { api_key: apiKey },
|
||||
});
|
||||
const token = await backendClient.getToken();
|
||||
const clients = createEvalClients({
|
||||
provider: modelProvider.provider,
|
||||
proxy: {
|
||||
baseURL: `${backendSettings.baseUrl}/api/w/${encodeURIComponent(proxyWorkspaceId)}/ai/proxy`,
|
||||
bearerToken: token,
|
||||
resourcePath,
|
||||
},
|
||||
}) as unknown as ChatClients;
|
||||
return await executeChatLoop(clients);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function toFrontendEvalProvider(
|
||||
|
||||
@@ -2,35 +2,9 @@ import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
buildProxyHeaders,
|
||||
buildProxyResourcePath,
|
||||
buildOpenAICompatibleClientOptions,
|
||||
resolveEvalModelProvider,
|
||||
} from "./providerConfig";
|
||||
|
||||
describe("buildOpenAICompatibleClientOptions", () => {
|
||||
it("adds Gemini's OpenAI-compatible base URL and client header", () => {
|
||||
const options = buildOpenAICompatibleClientOptions(
|
||||
"googleai",
|
||||
"gemini-test-key",
|
||||
);
|
||||
|
||||
expect(options).toMatchObject({
|
||||
apiKey: "gemini-test-key",
|
||||
baseURL: "https://generativelanguage.googleapis.com/v1beta/openai/",
|
||||
defaultHeaders: {
|
||||
"x-goog-api-client": "windmill-ai-evals/1.0",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps the default OpenAI-compatible config for OpenAI", () => {
|
||||
expect(
|
||||
buildOpenAICompatibleClientOptions("openai", "openai-test-key"),
|
||||
).toEqual({
|
||||
apiKey: "openai-test-key",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("proxy helpers", () => {
|
||||
it("builds provider-scoped proxy resource paths", () => {
|
||||
expect(buildProxyResourcePath("googleai")).toBe("f/evals/ai/googleai");
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import Anthropic from "@anthropic-ai/sdk";
|
||||
import OpenAI from "openai";
|
||||
import type { FrontendEvalModelConfig } from "../../../../core/models";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
|
||||
export type FrontendEvalProvider = FrontendEvalModelConfig["provider"];
|
||||
|
||||
@@ -15,15 +14,12 @@ export interface ResolvedEvalModelProvider {
|
||||
model: string;
|
||||
}
|
||||
|
||||
export interface EvalProxyClientConfig {
|
||||
export interface WindmillAiProxyClientConfig {
|
||||
baseURL: string;
|
||||
bearerToken: string;
|
||||
resourcePath: string;
|
||||
}
|
||||
|
||||
const GEMINI_OPENAI_BASE_URL =
|
||||
"https://generativelanguage.googleapis.com/v1beta/openai/";
|
||||
const GEMINI_GOOG_API_CLIENT = "windmill-ai-evals/1.0";
|
||||
const EVAL_PROXY_RESOURCE_PREFIX = "f/evals/ai";
|
||||
|
||||
export function buildProxyHeaders(
|
||||
@@ -40,25 +36,8 @@ export function buildProxyResourcePath(provider: FrontendEvalProvider): string {
|
||||
return `${EVAL_PROXY_RESOURCE_PREFIX}/${provider}`;
|
||||
}
|
||||
|
||||
export function buildOpenAICompatibleClientOptions(
|
||||
provider: Exclude<FrontendEvalProvider, "anthropic">,
|
||||
apiKey: string,
|
||||
): ConstructorParameters<typeof OpenAI>[0] {
|
||||
if (provider === "googleai") {
|
||||
return {
|
||||
apiKey,
|
||||
baseURL: GEMINI_OPENAI_BASE_URL,
|
||||
defaultHeaders: {
|
||||
"x-goog-api-client": GEMINI_GOOG_API_CLIENT,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return { apiKey };
|
||||
}
|
||||
|
||||
function buildProxyOpenAIClientOptions(
|
||||
proxy: EvalProxyClientConfig,
|
||||
proxy: WindmillAiProxyClientConfig,
|
||||
): ConstructorParameters<typeof OpenAI>[0] {
|
||||
return {
|
||||
apiKey: "unused",
|
||||
@@ -69,52 +48,24 @@ function buildProxyOpenAIClientOptions(
|
||||
|
||||
export function createEvalClients(input: {
|
||||
provider: FrontendEvalProvider;
|
||||
apiKey: string;
|
||||
transport?: FrontendEvalTransport;
|
||||
proxy?: EvalProxyClientConfig;
|
||||
proxy: WindmillAiProxyClientConfig;
|
||||
}): EvalClients {
|
||||
const transport = input.transport ?? "direct";
|
||||
|
||||
if (input.provider === "anthropic") {
|
||||
if (transport === "proxy") {
|
||||
if (!input.proxy) {
|
||||
throw new Error(
|
||||
"Missing proxy client configuration for proxy transport",
|
||||
);
|
||||
}
|
||||
return {
|
||||
openai: new OpenAI({ apiKey: "unused" }),
|
||||
anthropic: new Anthropic({
|
||||
apiKey: "unused",
|
||||
baseURL: input.proxy.baseURL,
|
||||
defaultHeaders: buildProxyHeaders(
|
||||
input.proxy.bearerToken,
|
||||
input.proxy.resourcePath,
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
openai: new OpenAI({ apiKey: "unused" }),
|
||||
anthropic: new Anthropic({ apiKey: input.apiKey }),
|
||||
};
|
||||
}
|
||||
|
||||
if (transport === "proxy") {
|
||||
if (!input.proxy) {
|
||||
throw new Error("Missing proxy client configuration for proxy transport");
|
||||
}
|
||||
return {
|
||||
openai: new OpenAI(buildProxyOpenAIClientOptions(input.proxy)),
|
||||
anthropic: new Anthropic({ apiKey: "unused" }),
|
||||
anthropic: new Anthropic({
|
||||
apiKey: "unused",
|
||||
baseURL: input.proxy.baseURL,
|
||||
defaultHeaders: buildProxyHeaders(
|
||||
input.proxy.bearerToken,
|
||||
input.proxy.resourcePath,
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
openai: new OpenAI(
|
||||
buildOpenAICompatibleClientOptions(input.provider, input.apiKey),
|
||||
),
|
||||
openai: new OpenAI(buildProxyOpenAIClientOptions(input.proxy)),
|
||||
anthropic: new Anthropic({ apiKey: "unused" }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import type { ChatCompletionMessageParam } from "openai/resources/chat/completions.mjs";
|
||||
import type { AIProvider } from "$lib/gen/types.gen";
|
||||
import type { FrontendEvalTransport } from "../../../../core/frontendTransport";
|
||||
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
|
||||
|
||||
export interface TokenUsage {
|
||||
@@ -15,14 +14,13 @@ export interface ToolCallDetail {
|
||||
}
|
||||
|
||||
export interface EvalRunnerOptions {
|
||||
backend: WindmillBackendSettings;
|
||||
maxIterations?: number;
|
||||
model?: string;
|
||||
workspace?: string;
|
||||
provider?: AIProvider;
|
||||
transport?: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
proxyCaseId?: string;
|
||||
proxyAttempt?: number;
|
||||
caseId?: string;
|
||||
attempt?: number;
|
||||
}
|
||||
|
||||
export interface RawEvalResult<TOutput> {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script'
|
||||
export type FrontendBenchmarkProgressSurface = 'flow' | 'app' | 'script' | 'global'
|
||||
|
||||
export type FrontendBenchmarkProgressEvent =
|
||||
| {
|
||||
|
||||
@@ -16,14 +16,13 @@ const FRONTEND_BENCHMARK_TEST =
|
||||
const FRONTEND_BENCHMARK_CONFIG =
|
||||
"../ai_evals/adapters/frontend/vitest.config.ts";
|
||||
|
||||
export type FrontendMode = "flow" | "app" | "script";
|
||||
export type FrontendMode = "flow" | "app" | "script" | "global";
|
||||
|
||||
export async function runFrontendBenchmarkAdapter(input: {
|
||||
mode: FrontendMode;
|
||||
caseIds: string[];
|
||||
runs: number;
|
||||
model?: string;
|
||||
transport?: string;
|
||||
verbose?: boolean;
|
||||
backendValidation?: string;
|
||||
}): Promise<BenchmarkRunResult> {
|
||||
@@ -44,10 +43,6 @@ export async function runFrontendBenchmarkAdapter(input: {
|
||||
WMILL_FRONTEND_AI_EVAL_BACKEND_VALIDATION: input.backendValidation ?? "",
|
||||
};
|
||||
|
||||
if (input.transport) {
|
||||
env.WMILL_FRONTEND_AI_EVAL_TRANSPORT = input.transport;
|
||||
}
|
||||
|
||||
try {
|
||||
await runVitestBenchmark(
|
||||
path.join(FRONTEND_DIR, "node_modules", ".bin", "vitest"),
|
||||
|
||||
@@ -65,6 +65,10 @@ vi.mock('$lib/gen', async () => {
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? (listBenchmarkScripts(data.workspace) ?? [])
|
||||
: actual.ScriptService.listScripts(data),
|
||||
existsScriptByPath: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? Boolean(getBenchmarkScriptByPath(data.workspace, data.path))
|
||||
: actual.ScriptService.existsScriptByPath(data),
|
||||
getScriptByPath: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
const script = getBenchmarkScriptByPath(data.workspace, data.path)
|
||||
@@ -91,6 +95,10 @@ vi.mock('$lib/gen', async () => {
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? (listBenchmarkFlows(data.workspace) ?? [])
|
||||
: actual.FlowService.listFlows(data),
|
||||
existsFlowByPath: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? Boolean(getBenchmarkFlowByPath(data.workspace, data.path))
|
||||
: actual.FlowService.existsFlowByPath(data),
|
||||
getFlowByPath: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
|
||||
@@ -142,6 +150,16 @@ vi.mock('$lib/gen', async () => {
|
||||
}
|
||||
}),
|
||||
ScheduleService: wrapService(actual.ScheduleService, {
|
||||
existsSchedule: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.ScheduleService.existsSchedule(data),
|
||||
listSchedules: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ScheduleService.listSchedules(data),
|
||||
getSchedule: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Schedule "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.ScheduleService.getSchedule(data)
|
||||
},
|
||||
previewSchedule: async (data: { requestBody?: Record<string, unknown> }) =>
|
||||
previewBenchmarkSchedule(data),
|
||||
createSchedule: async (data: { workspace: string; requestBody: Record<string, unknown> }) =>
|
||||
@@ -149,11 +167,167 @@ vi.mock('$lib/gen', async () => {
|
||||
? createBenchmarkSchedule(data)
|
||||
: actual.ScheduleService.createSchedule(data)
|
||||
}),
|
||||
ResourceService: wrapService(actual.ResourceService, {
|
||||
existsResource: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.ResourceService.existsResource(data),
|
||||
listResource: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.listResource(data),
|
||||
getResource: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Resource "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.ResourceService.getResource(data)
|
||||
},
|
||||
queryResourceTypes: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.ResourceService.queryResourceTypes(data)
|
||||
}),
|
||||
VariableService: wrapService(actual.VariableService, {
|
||||
existsVariable: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.VariableService.existsVariable(data),
|
||||
listVariable: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.VariableService.listVariable(data),
|
||||
getVariable: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Variable "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.VariableService.getVariable(data)
|
||||
}
|
||||
}),
|
||||
AppService: wrapService(actual.AppService, {
|
||||
existsApp: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.AppService.existsApp(data),
|
||||
listApps: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.AppService.listApps(data),
|
||||
getAppByPath: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`App "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.AppService.getAppByPath(data)
|
||||
}
|
||||
}),
|
||||
HttpTriggerService: wrapService(actual.HttpTriggerService, {
|
||||
existsHttpTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.HttpTriggerService.existsHttpTrigger(data),
|
||||
listHttpTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.HttpTriggerService.listHttpTriggers(data),
|
||||
getHttpTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`HTTP trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.HttpTriggerService.getHttpTrigger(data)
|
||||
},
|
||||
createHttpTrigger: async (data: { workspace: string; requestBody: Record<string, unknown> }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? createBenchmarkHttpTrigger(data)
|
||||
: actual.HttpTriggerService.createHttpTrigger(data)
|
||||
}),
|
||||
WebsocketTriggerService: wrapService(actual.WebsocketTriggerService, {
|
||||
existsWebsocketTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? false
|
||||
: actual.WebsocketTriggerService.existsWebsocketTrigger(data),
|
||||
listWebsocketTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? []
|
||||
: actual.WebsocketTriggerService.listWebsocketTriggers(data),
|
||||
getWebsocketTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Websocket trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.WebsocketTriggerService.getWebsocketTrigger(data)
|
||||
}
|
||||
}),
|
||||
KafkaTriggerService: wrapService(actual.KafkaTriggerService, {
|
||||
existsKafkaTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? false
|
||||
: actual.KafkaTriggerService.existsKafkaTrigger(data),
|
||||
listKafkaTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.KafkaTriggerService.listKafkaTriggers(data),
|
||||
getKafkaTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Kafka trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.KafkaTriggerService.getKafkaTrigger(data)
|
||||
}
|
||||
}),
|
||||
NatsTriggerService: wrapService(actual.NatsTriggerService, {
|
||||
existsNatsTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.NatsTriggerService.existsNatsTrigger(data),
|
||||
listNatsTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.NatsTriggerService.listNatsTriggers(data),
|
||||
getNatsTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`NATS trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.NatsTriggerService.getNatsTrigger(data)
|
||||
}
|
||||
}),
|
||||
PostgresTriggerService: wrapService(actual.PostgresTriggerService, {
|
||||
existsPostgresTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? false
|
||||
: actual.PostgresTriggerService.existsPostgresTrigger(data),
|
||||
listPostgresTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? []
|
||||
: actual.PostgresTriggerService.listPostgresTriggers(data),
|
||||
getPostgresTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Postgres trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.PostgresTriggerService.getPostgresTrigger(data)
|
||||
}
|
||||
}),
|
||||
MqttTriggerService: wrapService(actual.MqttTriggerService, {
|
||||
existsMqttTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.MqttTriggerService.existsMqttTrigger(data),
|
||||
listMqttTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.MqttTriggerService.listMqttTriggers(data),
|
||||
getMqttTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`MQTT trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.MqttTriggerService.getMqttTrigger(data)
|
||||
}
|
||||
}),
|
||||
SqsTriggerService: wrapService(actual.SqsTriggerService, {
|
||||
existsSqsTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.SqsTriggerService.existsSqsTrigger(data),
|
||||
listSqsTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.SqsTriggerService.listSqsTriggers(data),
|
||||
getSqsTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`SQS trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.SqsTriggerService.getSqsTrigger(data)
|
||||
}
|
||||
}),
|
||||
GcpTriggerService: wrapService(actual.GcpTriggerService, {
|
||||
existsGcpTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? false : actual.GcpTriggerService.existsGcpTrigger(data),
|
||||
listGcpTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.GcpTriggerService.listGcpTriggers(data),
|
||||
getGcpTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`GCP trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.GcpTriggerService.getGcpTrigger(data)
|
||||
}
|
||||
}),
|
||||
AzureTriggerService: wrapService(actual.AzureTriggerService, {
|
||||
existsAzureTrigger: async (data: { workspace: string; path: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace)
|
||||
? false
|
||||
: actual.AzureTriggerService.existsAzureTrigger(data),
|
||||
listAzureTriggers: async (data: { workspace: string }) =>
|
||||
hasBenchmarkWorkspace(data.workspace) ? [] : actual.AzureTriggerService.listAzureTriggers(data),
|
||||
getAzureTrigger: async (data: { workspace: string; path: string }) => {
|
||||
if (hasBenchmarkWorkspace(data.workspace)) {
|
||||
throw new Error(`Azure trigger "${data.path}" not found in benchmark workspace`)
|
||||
}
|
||||
return actual.AzureTriggerService.getAzureTrigger(data)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import type { WindmillBackendSettings } from "../../core/windmillBackendSettings";
|
||||
import {
|
||||
WindmillBackendClient,
|
||||
assertWindmillBackendReachable,
|
||||
} from "./windmillBackend";
|
||||
|
||||
const ORIGINAL_FETCH = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = ORIGINAL_FETCH;
|
||||
});
|
||||
|
||||
describe("assertWindmillBackendReachable", () => {
|
||||
it("logs in to verify backend reachability", async () => {
|
||||
const requests: Array<{ url: string; init?: RequestInit }> = [];
|
||||
globalThis.fetch = mockFetch(requests, textResponse(200, "token"));
|
||||
|
||||
await expect(
|
||||
assertWindmillBackendReachable(
|
||||
buildSettings({ baseUrl: "http://backend.test/reachable" }),
|
||||
),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(requests.map((entry) => entry.url)).toEqual([
|
||||
"http://backend.test/reachable/api/auth/login",
|
||||
]);
|
||||
});
|
||||
|
||||
it("adds setup guidance when the backend cannot be initialized", async () => {
|
||||
globalThis.fetch = mockFetch(
|
||||
[],
|
||||
textResponse(401, "invalid password"),
|
||||
);
|
||||
|
||||
await expect(
|
||||
assertWindmillBackendReachable(
|
||||
buildSettings({ baseUrl: "http://backend.test/auth-failure" }),
|
||||
),
|
||||
).rejects.toThrow(
|
||||
"Start a Windmill backend at that URL, or set WMILL_AI_EVAL_BACKEND_URL=<url>.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("WindmillBackendClient", () => {
|
||||
it("creates or reuses the specified backend workspace without deleting it", async () => {
|
||||
const requests: Array<{ url: string; init?: RequestInit }> = [];
|
||||
globalThis.fetch = mockFetch(
|
||||
requests,
|
||||
textResponse(200, "token"),
|
||||
textResponse(200, "false"),
|
||||
textResponse(200, ""),
|
||||
);
|
||||
|
||||
const client = new WindmillBackendClient(
|
||||
buildSettings({
|
||||
baseUrl: "http://backend.test/shared-workspace",
|
||||
workspaceOverride: "shared-evals",
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
client.withWorkspace("case-a", 1, async (workspaceId) => workspaceId),
|
||||
).resolves.toBe("shared-evals");
|
||||
|
||||
expect(requests.map((entry) => entry.url)).toEqual([
|
||||
"http://backend.test/shared-workspace/api/auth/login",
|
||||
"http://backend.test/shared-workspace/api/workspaces/exists",
|
||||
"http://backend.test/shared-workspace/api/workspaces/create",
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
function buildSettings(
|
||||
overrides: Partial<WindmillBackendSettings> = {},
|
||||
): WindmillBackendSettings {
|
||||
return {
|
||||
baseUrl: "http://backend.test/default",
|
||||
email: "admin@windmill.dev",
|
||||
password: "changeme",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function mockFetch(
|
||||
requests: Array<{ url: string; init?: RequestInit }>,
|
||||
...responses: Response[]
|
||||
): typeof fetch {
|
||||
const queue = [...responses];
|
||||
return async (input, init) => {
|
||||
const url = String(input);
|
||||
requests.push({ url, init });
|
||||
const next = queue.shift();
|
||||
if (!next) {
|
||||
throw new Error(`Unexpected fetch: ${url}`);
|
||||
}
|
||||
return next;
|
||||
};
|
||||
}
|
||||
|
||||
function textResponse(status: number, body: string): Response {
|
||||
return new Response(body, { status });
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import type { WindmillBackendSettings } from "../../core/windmillBackendSettings
|
||||
|
||||
const tokenCache = new Map<string, Promise<string>>();
|
||||
const sharedWorkspaceQueue = new Map<string, Promise<void>>();
|
||||
const DEFAULT_WORKSPACE_PREFIX = "ai-evals";
|
||||
|
||||
export class WindmillBackendClient {
|
||||
constructor(private readonly settings: WindmillBackendSettings) {}
|
||||
@@ -14,7 +15,7 @@ export class WindmillBackendClient {
|
||||
): Promise<T> {
|
||||
const workspaceId =
|
||||
this.settings.workspaceOverride ??
|
||||
buildWorkspaceId(this.settings.workspacePrefix, caseId, attempt);
|
||||
buildWorkspaceId(caseId, attempt);
|
||||
|
||||
const run = async () => {
|
||||
await this.ensureWorkspace(workspaceId);
|
||||
@@ -22,7 +23,7 @@ export class WindmillBackendClient {
|
||||
try {
|
||||
return await body(workspaceId);
|
||||
} finally {
|
||||
if (!this.settings.keepWorkspaces && !this.settings.workspaceOverride) {
|
||||
if (!this.settings.workspaceOverride) {
|
||||
await this.deleteWorkspace(workspaceId).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
@@ -136,6 +137,24 @@ export class WindmillBackendClient {
|
||||
}
|
||||
}
|
||||
|
||||
export async function assertWindmillBackendReachable(
|
||||
settings: WindmillBackendSettings,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await new WindmillBackendClient(settings).getToken();
|
||||
} catch (error) {
|
||||
const details = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(
|
||||
[
|
||||
`Could not initialize the Windmill backend for AI eval proxy at ${settings.baseUrl}.`,
|
||||
"Start a Windmill backend at that URL, or set WMILL_AI_EVAL_BACKEND_URL=<url>.",
|
||||
`Using login ${settings.email}; if authentication failed, set WMILL_AI_EVAL_BACKEND_EMAIL and WMILL_AI_EVAL_BACKEND_PASSWORD.`,
|
||||
`Details: ${details}`,
|
||||
].join("\n"),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function withSharedWorkspaceLock<T>(
|
||||
workspaceId: string,
|
||||
body: () => Promise<T>,
|
||||
@@ -160,18 +179,14 @@ async function withSharedWorkspaceLock<T>(
|
||||
}
|
||||
}
|
||||
|
||||
function buildWorkspaceId(
|
||||
prefix: string,
|
||||
caseId: string,
|
||||
attempt: number,
|
||||
): string {
|
||||
function buildWorkspaceId(caseId: string, attempt: number): string {
|
||||
const caseSlug = caseId
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.slice(0, 30);
|
||||
const suffix = randomUUID().slice(0, 8);
|
||||
return `${prefix}-${caseSlug || "case"}-a${attempt}-${suffix}`;
|
||||
return `${DEFAULT_WORKSPACE_PREFIX}-${caseSlug || "case"}-a${attempt}-${suffix}`;
|
||||
}
|
||||
|
||||
async function expectOk(response: Response, context: string): Promise<void> {
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
- id: global-test1-script-create
|
||||
prompt: |-
|
||||
Create a draft Bun script at `f/evals/global/greet_user`.
|
||||
It should take a string `name` input and return `Hello, ${name}!`.
|
||||
Leave it as an AI draft only; do not deploy or save it.
|
||||
runtime:
|
||||
maxTurns: 8
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: script
|
||||
path: f/evals/global/greet_user
|
||||
language: bun
|
||||
valueIncludes:
|
||||
- name
|
||||
- Hello
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_script
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- creates a Bun script draft at f/evals/global/greet_user
|
||||
- the script accepts a name input
|
||||
- the script returns a greeting containing Hello, the provided name, and an exclamation mark
|
||||
- the result stays as an AI draft and is not deployed or saved to the workspace
|
||||
|
||||
- id: global-test2-script-edit-existing
|
||||
prompt: |-
|
||||
Update the existing workspace script at `f/evals/global/format_greeting`.
|
||||
Keep it as a Bun script, but change the greeting so the provided name is uppercased and the returned message ends with an exclamation mark.
|
||||
Leave the result as an AI draft only; do not deploy or save it.
|
||||
initial: ai_evals/fixtures/frontend/global/initial/format_greeting_script.json
|
||||
runtime:
|
||||
maxTurns: 8
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: script
|
||||
path: f/evals/global/format_greeting
|
||||
language: bun
|
||||
valueIncludes:
|
||||
- toUpperCase
|
||||
- "!"
|
||||
toolExpect:
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
judgeChecklist:
|
||||
- creates an AI draft for the existing f/evals/global/format_greeting script
|
||||
- preserves the script as Bun
|
||||
- uppercases the provided name in the greeting
|
||||
- returns a message ending with an exclamation mark
|
||||
- does not deploy or save the draft to the workspace
|
||||
|
||||
- id: global-test3-flow-create
|
||||
prompt: |-
|
||||
Create a draft flow at `f/evals/global/sum_numbers`.
|
||||
It should take two numeric inputs, `a` and `b`, and return their sum.
|
||||
Leave it as an AI draft only; do not deploy or save it.
|
||||
runtime:
|
||||
maxTurns: 8
|
||||
validate:
|
||||
draftCountExactly: 1
|
||||
requiredDrafts:
|
||||
- type: flow
|
||||
path: f/evals/global/sum_numbers
|
||||
valueIncludes:
|
||||
- modules
|
||||
- rawscript
|
||||
- flow_input.a
|
||||
- flow_input.b
|
||||
toolExpect:
|
||||
requiredToolsUsed:
|
||||
- write_flow
|
||||
forbiddenToolsUsed:
|
||||
- deploy_workspace_item
|
||||
- delete_workspace_item
|
||||
toolCallArgs:
|
||||
- tool: write_flow
|
||||
field: modules
|
||||
stringStartsWithAnyOf:
|
||||
- "["
|
||||
judgeChecklist:
|
||||
- creates a flow draft at f/evals/global/sum_numbers
|
||||
- the flow accepts numeric inputs a and b
|
||||
- the flow returns the sum of a and b
|
||||
- the result stays as an AI draft and is not deployed or saved to the workspace
|
||||
+9
-23
@@ -27,11 +27,8 @@ import { EVAL_MODES, type EvalMode } from "../core/types";
|
||||
import { DEFAULT_JUDGE_MODEL } from "../core/judge";
|
||||
import { createCliModeRunner } from "../modes/cli";
|
||||
import { runFrontendBenchmarkAdapter } from "../adapters/frontend/runtime";
|
||||
import {
|
||||
FRONTEND_EVAL_TRANSPORTS,
|
||||
type FrontendEvalTransport,
|
||||
parseFrontendEvalTransport,
|
||||
} from "../core/frontendTransport";
|
||||
import { resolveWindmillBackendSettings } from "../core/windmillBackendSettings";
|
||||
import { assertWindmillBackendReachable } from "../adapters/frontend/windmillBackend";
|
||||
|
||||
async function main() {
|
||||
const program = new Command()
|
||||
@@ -56,6 +53,7 @@ async function main() {
|
||||
" bun run cli -- run flow --record",
|
||||
" bun run cli -- run flow --backend-validation preview",
|
||||
" bun run cli -- run flow flow-test5-simple-modification --runs 3",
|
||||
" bun run cli -- run global global-test1-script-create",
|
||||
" bun run cli -- run cli bun-hello-script",
|
||||
"",
|
||||
"Models:",
|
||||
@@ -73,7 +71,7 @@ async function main() {
|
||||
program
|
||||
.command("cases")
|
||||
.description("List available cases")
|
||||
.argument("[mode]", "cli, flow, script, or app", parseOptionalMode)
|
||||
.argument("[mode]", "cli, flow, script, app, or global", parseOptionalMode)
|
||||
.action(async (mode?: EvalMode) => {
|
||||
await handleCases(mode);
|
||||
});
|
||||
@@ -81,7 +79,7 @@ async function main() {
|
||||
program
|
||||
.command("run")
|
||||
.description("Run one benchmark mode")
|
||||
.argument("<mode>", "cli, flow, script, or app", parseMode)
|
||||
.argument("<mode>", "cli, flow, script, app, or global", parseMode)
|
||||
.argument("[caseIds...]", "specific case ids to run")
|
||||
.option(
|
||||
"--runs <n>",
|
||||
@@ -98,10 +96,6 @@ async function main() {
|
||||
"--models <names>",
|
||||
"comma-separated model aliases to run sequentially",
|
||||
)
|
||||
.option(
|
||||
"--transport <mode>",
|
||||
`frontend transport (${FRONTEND_EVAL_TRANSPORTS.join(", ")})`,
|
||||
)
|
||||
.option("--verbose", "stream assistant output during frontend runs")
|
||||
.option(
|
||||
"--record",
|
||||
@@ -120,7 +114,6 @@ async function main() {
|
||||
output?: string;
|
||||
model?: string;
|
||||
models?: string;
|
||||
transport?: string;
|
||||
verbose?: boolean;
|
||||
record?: boolean;
|
||||
backendValidation?: string;
|
||||
@@ -133,9 +126,6 @@ async function main() {
|
||||
outputPath: options.output,
|
||||
model: options.model,
|
||||
models: options.models,
|
||||
transport: options.transport
|
||||
? parseFrontendEvalTransport(options.transport)
|
||||
: undefined,
|
||||
verbose: options.verbose ?? false,
|
||||
record: options.record ?? false,
|
||||
backendValidation: options.backendValidation,
|
||||
@@ -163,7 +153,7 @@ function handleModels() {
|
||||
process.stdout.write("Available models\n");
|
||||
for (const model of EVAL_MODELS) {
|
||||
const supports = [
|
||||
...(model.frontend ? ["flow", "script", "app"] : []),
|
||||
...(model.frontend ? ["flow", "script", "app", "global"] : []),
|
||||
...(model.cli ? ["cli"] : []),
|
||||
];
|
||||
const aliases = [
|
||||
@@ -184,7 +174,6 @@ async function handleRun(input: {
|
||||
outputPath?: string;
|
||||
model?: string;
|
||||
models?: string;
|
||||
transport?: FrontendEvalTransport;
|
||||
verbose: boolean;
|
||||
record: boolean;
|
||||
backendValidation?: string;
|
||||
@@ -197,11 +186,6 @@ async function handleRun(input: {
|
||||
if (input.model && input.models) {
|
||||
throw new Error("Use either --model or --models, not both");
|
||||
}
|
||||
if (input.mode === "cli" && input.transport === "proxy") {
|
||||
throw new Error(
|
||||
"--transport proxy is only supported for flow, script, and app modes",
|
||||
);
|
||||
}
|
||||
|
||||
const selectedCases = await loadSelectedCases(input.mode, input.caseIds);
|
||||
const models = resolveRequestedModels(input.mode, input.model, input.models);
|
||||
@@ -220,6 +204,9 @@ async function handleRun(input: {
|
||||
"--backend-validation currently supports only flow and script modes",
|
||||
);
|
||||
}
|
||||
if (input.mode !== "cli") {
|
||||
await assertWindmillBackendReachable(resolveWindmillBackendSettings());
|
||||
}
|
||||
|
||||
const summaries: Array<{
|
||||
label: string;
|
||||
@@ -249,7 +236,6 @@ async function handleRun(input: {
|
||||
caseIds: input.caseIds,
|
||||
runs: input.runs,
|
||||
model: model.id,
|
||||
transport: input.transport,
|
||||
verbose: input.verbose,
|
||||
backendValidation,
|
||||
});
|
||||
|
||||
@@ -13,9 +13,7 @@ export interface BackendValidationSettings {
|
||||
baseUrl: string;
|
||||
email: string;
|
||||
password: string;
|
||||
keepWorkspaces: boolean;
|
||||
workspaceOverride?: string;
|
||||
workspacePrefix: string;
|
||||
pollIntervalMs: number;
|
||||
maxWaitMs: number;
|
||||
}
|
||||
|
||||
@@ -183,6 +183,26 @@ describe("loadCases", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("loads global draft validation and forbidden tool expectations", async () => {
|
||||
const globalCases = await loadCases("global");
|
||||
const caseEntry = globalCases.find((entry) => entry.id === "global-test1-script-create");
|
||||
|
||||
expect(caseEntry?.validate).toMatchObject({
|
||||
draftCountExactly: 1,
|
||||
requiredDrafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(caseEntry?.toolExpect).toMatchObject({
|
||||
requiredToolsUsed: ["write_script"],
|
||||
forbiddenToolsUsed: ["deploy_workspace_item", "delete_workspace_item"],
|
||||
});
|
||||
});
|
||||
|
||||
it("loads tool expectations for workspace mutation cases", async () => {
|
||||
const scriptCases = await loadCases("script");
|
||||
const caseEntry = scriptCases.find(
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import {
|
||||
parseFrontendEvalTransport,
|
||||
resolveFrontendEvalTransportSettings,
|
||||
} from "./frontendTransport";
|
||||
|
||||
const ORIGINAL_ENV = {
|
||||
WMILL_AI_EVAL_BACKEND_URL: process.env.WMILL_AI_EVAL_BACKEND_URL,
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
if (ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL === undefined) {
|
||||
delete process.env.WMILL_AI_EVAL_BACKEND_URL;
|
||||
} else {
|
||||
process.env.WMILL_AI_EVAL_BACKEND_URL =
|
||||
ORIGINAL_ENV.WMILL_AI_EVAL_BACKEND_URL;
|
||||
}
|
||||
});
|
||||
|
||||
describe("parseFrontendEvalTransport", () => {
|
||||
it("defaults to direct when unset", () => {
|
||||
expect(parseFrontendEvalTransport(undefined)).toBe("direct");
|
||||
});
|
||||
|
||||
it("accepts proxy explicitly", () => {
|
||||
expect(parseFrontendEvalTransport("proxy")).toBe("proxy");
|
||||
});
|
||||
|
||||
it("rejects unsupported values", () => {
|
||||
expect(() => parseFrontendEvalTransport("worker")).toThrow(
|
||||
"Unsupported frontend eval transport: worker",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("resolveFrontendEvalTransportSettings", () => {
|
||||
it("includes backend settings for proxy transport", () => {
|
||||
process.env.WMILL_AI_EVAL_BACKEND_URL = "http://127.0.0.1:8000/";
|
||||
|
||||
expect(
|
||||
resolveFrontendEvalTransportSettings({
|
||||
evalMode: "app",
|
||||
requestedTransport: "proxy",
|
||||
}),
|
||||
).toMatchObject({
|
||||
transport: "proxy",
|
||||
backend: {
|
||||
baseUrl: "http://127.0.0.1:8000",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps direct transport for cli runs", () => {
|
||||
expect(
|
||||
resolveFrontendEvalTransportSettings({
|
||||
evalMode: "cli",
|
||||
requestedTransport: "direct",
|
||||
}),
|
||||
).toEqual({
|
||||
transport: "direct",
|
||||
backend: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,49 +0,0 @@
|
||||
import type { EvalMode } from "./types";
|
||||
import type { WindmillBackendSettings } from "./windmillBackendSettings";
|
||||
import { resolveWindmillBackendSettings } from "./windmillBackendSettings";
|
||||
|
||||
export const FRONTEND_EVAL_TRANSPORTS = ["direct", "proxy"] as const;
|
||||
|
||||
export type FrontendEvalTransport = (typeof FRONTEND_EVAL_TRANSPORTS)[number];
|
||||
|
||||
export interface FrontendEvalTransportSettings {
|
||||
transport: FrontendEvalTransport;
|
||||
backend?: WindmillBackendSettings;
|
||||
}
|
||||
|
||||
export function parseFrontendEvalTransport(
|
||||
value?: string | null,
|
||||
): FrontendEvalTransport {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
|
||||
if (!normalized || normalized === "direct") {
|
||||
return "direct";
|
||||
}
|
||||
|
||||
if (normalized === "proxy") {
|
||||
return "proxy";
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`Unsupported frontend eval transport: ${value}. Use one of: ${FRONTEND_EVAL_TRANSPORTS.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
export function resolveFrontendEvalTransportSettings(input: {
|
||||
evalMode: EvalMode;
|
||||
requestedTransport?: string | null;
|
||||
}): FrontendEvalTransportSettings {
|
||||
const transport = parseFrontendEvalTransport(input.requestedTransport);
|
||||
|
||||
if (transport === "proxy" && input.evalMode === "cli") {
|
||||
throw new Error(
|
||||
'Frontend eval transport "proxy" is only supported for flow, script, and app evals',
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
transport,
|
||||
backend:
|
||||
transport === "proxy" ? resolveWindmillBackendSettings() : undefined,
|
||||
};
|
||||
}
|
||||
@@ -145,7 +145,7 @@ export function resolveEvalModel(mode: EvalMode, alias?: string): EvalModelSpec
|
||||
export function getEvalModelHelpText(): string {
|
||||
return EVAL_MODELS.map((model) => {
|
||||
const modes = [
|
||||
...(model.frontend ? ["flow", "script", "app"] : []),
|
||||
...(model.frontend ? ["flow", "script", "app", "global"] : []),
|
||||
...(model.cli ? ["cli"] : []),
|
||||
];
|
||||
return ` ${model.id.padEnd(8)} ${model.label} (${modes.join(", ")})`;
|
||||
|
||||
@@ -74,7 +74,6 @@ export function buildRunResult(input: {
|
||||
mode: EvalMode;
|
||||
runs: number;
|
||||
runModel: string | null;
|
||||
transport?: BenchmarkRunResult["transport"];
|
||||
judgeModel: string | null;
|
||||
caseResults: BenchmarkCaseResult[];
|
||||
}): BenchmarkRunResult {
|
||||
@@ -116,7 +115,6 @@ export function buildRunResult(input: {
|
||||
gitSha: getGitSha(),
|
||||
runs: input.runs,
|
||||
runModel: input.runModel,
|
||||
transport: input.transport ?? null,
|
||||
judgeModel: input.judgeModel,
|
||||
caseCount: input.caseResults.length,
|
||||
attemptCount,
|
||||
@@ -142,9 +140,6 @@ export function formatRunSummary(result: BenchmarkRunResult): string {
|
||||
`Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
|
||||
`Average duration: ${Math.round(result.averageDurationMs)}ms`,
|
||||
];
|
||||
if (result.transport) {
|
||||
lines.splice(1, 0, `Transport: ${result.transport}`);
|
||||
}
|
||||
|
||||
const failures = collectFailures(result);
|
||||
if (failures.length > 0) {
|
||||
@@ -251,7 +246,6 @@ function toHistoryRecord(result: BenchmarkRunResult) {
|
||||
mode: result.mode,
|
||||
runs: result.runs,
|
||||
runModel: result.runModel,
|
||||
transport: result.transport,
|
||||
judgeModel: result.judgeModel,
|
||||
caseCount: result.caseCount,
|
||||
attemptCount: result.attemptCount,
|
||||
|
||||
+24
-4
@@ -1,7 +1,6 @@
|
||||
export const EVAL_MODES = ["cli", "flow", "script", "app"] as const;
|
||||
export const EVAL_MODES = ["cli", "flow", "script", "app", "global"] as const;
|
||||
|
||||
export type EvalMode = (typeof EVAL_MODES)[number];
|
||||
export type FrontendEvalTransport = "direct" | "proxy";
|
||||
|
||||
export interface EvalCaseRuntimeBackendPreview {
|
||||
args?: Record<string, unknown>;
|
||||
@@ -109,6 +108,27 @@ export interface AppValidationSpec {
|
||||
forbiddenAppContent?: string[];
|
||||
}
|
||||
|
||||
export interface GlobalDraftRequirement {
|
||||
type: string;
|
||||
path: string;
|
||||
triggerKind?: string;
|
||||
language?: string;
|
||||
summaryIncludes?: string[];
|
||||
valueIncludes?: string[];
|
||||
valueExcludes?: string[];
|
||||
}
|
||||
|
||||
export interface GlobalValidationSpec {
|
||||
draftCountAtLeast?: number;
|
||||
draftCountExactly?: number;
|
||||
requiredDrafts?: GlobalDraftRequirement[];
|
||||
forbiddenDrafts?: Array<{
|
||||
type: string;
|
||||
path: string;
|
||||
triggerKind?: string;
|
||||
}>;
|
||||
}
|
||||
|
||||
export interface CliValidationSpec {
|
||||
requiredSkills?: string[];
|
||||
forbiddenSkills?: string[];
|
||||
@@ -137,10 +157,11 @@ export interface ToolCallArgumentRule {
|
||||
|
||||
export interface ToolValidationSpec {
|
||||
requiredToolsUsed?: string[];
|
||||
forbiddenToolsUsed?: string[];
|
||||
toolCallArgs?: ToolCallArgumentRule[];
|
||||
}
|
||||
|
||||
export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec;
|
||||
export type EvalValidationSpec = FlowValidationSpec | AppValidationSpec | GlobalValidationSpec;
|
||||
|
||||
export interface EvalCase {
|
||||
id: string;
|
||||
@@ -297,7 +318,6 @@ export interface BenchmarkRunResult {
|
||||
gitSha: string | null;
|
||||
runs: number;
|
||||
runModel: string | null;
|
||||
transport: FrontendEvalTransport | null;
|
||||
judgeModel: string | null;
|
||||
caseCount: number;
|
||||
attemptCount: number;
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from "bun:test";
|
||||
import {
|
||||
validateAppState,
|
||||
validateCliWorkspace,
|
||||
validateGlobalState,
|
||||
validateScriptState,
|
||||
validateToolExpectations,
|
||||
} from "./validators";
|
||||
@@ -117,6 +118,230 @@ describe("validateToolExpectations", () => {
|
||||
details: 'rejected prefixes: schedules/; values: "schedules/greet_user_daily"',
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects forbidden tool usage", () => {
|
||||
const checks = validateToolExpectations({
|
||||
run: {
|
||||
success: true,
|
||||
actual: {},
|
||||
assistantMessageCount: 1,
|
||||
toolCallCount: 1,
|
||||
toolsUsed: ["write_script", "deploy_workspace_item"],
|
||||
skillsInvoked: [],
|
||||
},
|
||||
toolExpect: {
|
||||
forbiddenToolsUsed: ["deploy_workspace_item"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks).toContainEqual({
|
||||
name: "does not use deploy_workspace_item",
|
||||
passed: false,
|
||||
details: "tools used: write_script, deploy_workspace_item",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateGlobalState", () => {
|
||||
it("accepts a required script draft", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
validate: {
|
||||
draftCountExactly: 1,
|
||||
requiredDrafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
valueIncludes: ["Hello"],
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks.every((check) => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it("fails when a required draft is missing", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [],
|
||||
},
|
||||
validate: {
|
||||
requiredDrafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks).toContainEqual({
|
||||
name: "global includes script draft f/evals/global/greet_user",
|
||||
passed: false,
|
||||
details: "drafts: none",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_python",
|
||||
language: "python3",
|
||||
value: "def main(name: str):\n return f'Hello, {name}!'\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks.some((check) => check.name.includes("exports entrypoint"))).toBe(
|
||||
false
|
||||
);
|
||||
expect(checks.every((check) => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it("allows read-only global cases without draft expectations", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [],
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
checks.some(
|
||||
(check) => check.name === "global produced at least one draft"
|
||||
)
|
||||
).toBe(false);
|
||||
expect(checks.every((check) => check.passed)).toBe(true);
|
||||
});
|
||||
|
||||
it("matches expected global draft fixtures", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
value:
|
||||
"export async function main(name: string) {\r\n return `Hello, ${name}!`\r\n}\r\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
expected: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(checks).toContainEqual({
|
||||
name: "global drafts match expected",
|
||||
passed: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("fails when expected global draft fixtures differ", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
expected: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Bonjour, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const expectedMatchCheck = checks.find(
|
||||
(check) => check.name === "global drafts match expected"
|
||||
);
|
||||
expect(expectedMatchCheck?.passed).toBe(false);
|
||||
expect(expectedMatchCheck?.details).toContain(
|
||||
"script:f/evals/global/greet_user value differs"
|
||||
);
|
||||
expect(expectedMatchCheck?.details).toContain("Hello");
|
||||
expect(expectedMatchCheck?.details).toContain("Bonjour");
|
||||
});
|
||||
|
||||
it("explains expected global draft metadata mismatches", () => {
|
||||
const checks = validateGlobalState({
|
||||
actual: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "bun",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
expected: {
|
||||
drafts: [
|
||||
{
|
||||
type: "script",
|
||||
path: "f/evals/global/greet_user",
|
||||
language: "python3",
|
||||
value:
|
||||
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
|
||||
isDraft: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const expectedMatchCheck = checks.find(
|
||||
(check) => check.name === "global drafts match expected"
|
||||
);
|
||||
expect(expectedMatchCheck?.passed).toBe(false);
|
||||
expect(expectedMatchCheck?.details).toContain(
|
||||
"script:f/evals/global/greet_user language differs"
|
||||
);
|
||||
expect(expectedMatchCheck?.details).toContain('actual="bun"');
|
||||
expect(expectedMatchCheck?.details).toContain('expected="python3"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("validateAppState", () => {
|
||||
|
||||
@@ -6,6 +6,7 @@ import type {
|
||||
CliTrace,
|
||||
CliValidationSpec,
|
||||
FlowValidationSpec,
|
||||
GlobalValidationSpec,
|
||||
ModeRunOutput,
|
||||
ToolValidationSpec,
|
||||
} from "./types";
|
||||
@@ -51,6 +52,20 @@ export interface AppDatatableState {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface GlobalDraftState {
|
||||
drafts: GlobalDraft[];
|
||||
}
|
||||
|
||||
export interface GlobalDraft {
|
||||
type: string;
|
||||
path: string;
|
||||
triggerKind?: string;
|
||||
summary?: string;
|
||||
language?: string;
|
||||
value?: unknown;
|
||||
isDraft?: boolean;
|
||||
}
|
||||
|
||||
const TS_LIKE_LANGUAGES = new Set(["bun", "deno", "nativets", "bunnative", "ts", "typescript"]);
|
||||
const CONTROL_FLOW_MODULE_TYPES = new Set(["branchone", "branchall", "forloopflow", "whileloopflow"]);
|
||||
|
||||
@@ -154,6 +169,16 @@ export function validateToolExpectations(input: {
|
||||
);
|
||||
}
|
||||
|
||||
for (const toolName of expect.forbiddenToolsUsed ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`does not use ${toolName}`,
|
||||
!input.run.toolsUsed.includes(toolName),
|
||||
`tools used: ${input.run.toolsUsed.join(", ") || "none"}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
for (const rule of expect.toolCallArgs ?? []) {
|
||||
const calls = toolCallDetails.filter((call) => call.name === rule.tool);
|
||||
checks.push(
|
||||
@@ -202,6 +227,161 @@ export function validateToolExpectations(input: {
|
||||
return checks;
|
||||
}
|
||||
|
||||
export function validateGlobalState(input: {
|
||||
actual: GlobalDraftState;
|
||||
expected?: GlobalDraftState;
|
||||
validate?: GlobalValidationSpec;
|
||||
}): BenchmarkCheck[] {
|
||||
const drafts = input.actual.drafts ?? [];
|
||||
const checks: BenchmarkCheck[] = [];
|
||||
|
||||
// Read-only global cases are valid; only enforce draft production when the
|
||||
// case explicitly asks for draft output.
|
||||
if (globalValidationExpectsDrafts(input)) {
|
||||
checks.push(
|
||||
check(
|
||||
"global produced at least one draft",
|
||||
drafts.length > 0,
|
||||
`drafts=${drafts.length}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
checks.push(
|
||||
check(
|
||||
"all global outputs are drafts",
|
||||
drafts.every((draft) => draft.isDraft === true),
|
||||
summarizeGlobalDrafts(drafts)
|
||||
)
|
||||
);
|
||||
|
||||
for (const draft of drafts) {
|
||||
if (draft.type !== "script" || typeof draft.value !== "string") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const language = (draft.language ?? "bun").toLowerCase();
|
||||
const syntaxErrors = getScriptSyntaxErrors(draft.value, language);
|
||||
if (TS_LIKE_LANGUAGES.has(language)) {
|
||||
checks.push(
|
||||
check(
|
||||
`script draft ${draft.path} exports entrypoint`,
|
||||
hasSupportedEntrypoint(draft.value)
|
||||
)
|
||||
);
|
||||
}
|
||||
checks.push(
|
||||
check(
|
||||
`script draft ${draft.path} has no syntax errors`,
|
||||
syntaxErrors.length === 0,
|
||||
summarizeProblems(syntaxErrors)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (input.expected) {
|
||||
checks.push(
|
||||
check(
|
||||
"global drafts match expected",
|
||||
globalDraftStatesEqual(input.actual, input.expected),
|
||||
describeGlobalDraftStateMismatch(input.actual, input.expected)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const validate = input.validate;
|
||||
if (!validate) {
|
||||
return checks;
|
||||
}
|
||||
|
||||
if (validate.draftCountAtLeast !== undefined) {
|
||||
checks.push(
|
||||
check(
|
||||
`global includes at least ${validate.draftCountAtLeast} draft(s)`,
|
||||
drafts.length >= validate.draftCountAtLeast,
|
||||
`drafts=${drafts.length}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
if (validate.draftCountExactly !== undefined) {
|
||||
checks.push(
|
||||
check(
|
||||
`global includes exactly ${validate.draftCountExactly} draft(s)`,
|
||||
drafts.length === validate.draftCountExactly,
|
||||
`drafts=${drafts.length}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
for (const required of validate.requiredDrafts ?? []) {
|
||||
const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind);
|
||||
checks.push(
|
||||
check(
|
||||
`global includes ${required.type} draft ${required.path}`,
|
||||
Boolean(draft),
|
||||
summarizeGlobalDrafts(drafts)
|
||||
)
|
||||
);
|
||||
if (!draft) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (required.language !== undefined) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} uses ${required.language}`,
|
||||
draft.language === required.language,
|
||||
`language=${draft.language ?? "(none)"}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
for (const snippet of required.summaryIncludes ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} summary includes '${snippet}'`,
|
||||
normalizeText(draft.summary ?? "").includes(normalizeText(snippet)),
|
||||
`summary=${draft.summary ?? ""}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
const valueText = stringifyGlobalDraftValue(draft.value);
|
||||
for (const snippet of required.valueIncludes ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} value includes '${snippet}'`,
|
||||
normalizeText(valueText).includes(normalizeText(snippet)),
|
||||
truncateForDetails(valueText)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
for (const snippet of required.valueExcludes ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`${required.type} draft ${required.path} value excludes '${snippet}'`,
|
||||
!normalizeText(valueText).includes(normalizeText(snippet)),
|
||||
truncateForDetails(valueText)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const forbidden of validate.forbiddenDrafts ?? []) {
|
||||
checks.push(
|
||||
check(
|
||||
`global does not include ${forbidden.type} draft ${forbidden.path}`,
|
||||
!findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind),
|
||||
summarizeGlobalDrafts(drafts)
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
return checks;
|
||||
}
|
||||
|
||||
export function validateAppState(input: {
|
||||
actual: AppFilesState;
|
||||
initial?: AppFilesState;
|
||||
@@ -433,6 +613,202 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined {
|
||||
return `${problems.slice(0, limit).join("; ")}; ...and ${problems.length - limit} more`;
|
||||
}
|
||||
|
||||
function findGlobalDraft(
|
||||
drafts: GlobalDraft[],
|
||||
type: string,
|
||||
path: string,
|
||||
triggerKind?: string
|
||||
): GlobalDraft | undefined {
|
||||
return drafts.find(
|
||||
(draft) =>
|
||||
draft.type === type &&
|
||||
draft.path === path &&
|
||||
(triggerKind === undefined || draft.triggerKind === triggerKind)
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeGlobalDrafts(drafts: GlobalDraft[]): string {
|
||||
const summary = drafts
|
||||
.map((draft) => formatGlobalDraftKey(draft))
|
||||
.join(", ");
|
||||
return `drafts: ${summary || "none"}`;
|
||||
}
|
||||
|
||||
function formatGlobalDraftKey(draft: GlobalDraft): string {
|
||||
return `${draft.type}${draft.triggerKind ? `:${draft.triggerKind}` : ""}:${draft.path}`;
|
||||
}
|
||||
|
||||
function globalValidationExpectsDrafts(input: {
|
||||
expected?: GlobalDraftState;
|
||||
validate?: GlobalValidationSpec;
|
||||
}): boolean {
|
||||
const validate = input.validate;
|
||||
return (
|
||||
(input.expected?.drafts?.length ?? 0) > 0 ||
|
||||
(validate?.requiredDrafts?.length ?? 0) > 0 ||
|
||||
(validate?.draftCountAtLeast ?? 0) > 0 ||
|
||||
(validate?.draftCountExactly ?? 0) > 0
|
||||
);
|
||||
}
|
||||
|
||||
function globalDraftStatesEqual(left: GlobalDraftState, right: GlobalDraftState): boolean {
|
||||
return (
|
||||
JSON.stringify(canonicalizeGlobalDrafts(left.drafts ?? [])) ===
|
||||
JSON.stringify(canonicalizeGlobalDrafts(right.drafts ?? []))
|
||||
);
|
||||
}
|
||||
|
||||
function describeGlobalDraftStateMismatch(
|
||||
actual: GlobalDraftState,
|
||||
expected: GlobalDraftState
|
||||
): string {
|
||||
const actualDrafts = actual.drafts ?? [];
|
||||
const expectedDrafts = expected.drafts ?? [];
|
||||
const actualByKey = new Map(
|
||||
actualDrafts.map((draft) => [globalDraftSortKey(draft), draft] as const)
|
||||
);
|
||||
const expectedByKey = new Map(
|
||||
expectedDrafts.map((draft) => [globalDraftSortKey(draft), draft] as const)
|
||||
);
|
||||
|
||||
for (const key of Array.from(expectedByKey.keys()).sort()) {
|
||||
const expectedDraft = expectedByKey.get(key);
|
||||
if (expectedDraft && !actualByKey.has(key)) {
|
||||
return `missing expected draft ${formatGlobalDraftKey(expectedDraft)}; actual=${summarizeGlobalDrafts(actualDrafts)}`;
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of Array.from(actualByKey.keys()).sort()) {
|
||||
const actualDraft = actualByKey.get(key);
|
||||
if (actualDraft && !expectedByKey.has(key)) {
|
||||
return `unexpected draft ${formatGlobalDraftKey(actualDraft)}; expected=${summarizeGlobalDrafts(expectedDrafts)}`;
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of Array.from(expectedByKey.keys()).sort()) {
|
||||
const actualDraft = actualByKey.get(key);
|
||||
const expectedDraft = expectedByKey.get(key);
|
||||
if (!actualDraft || !expectedDraft) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fieldMismatch = describeGlobalDraftFieldMismatch(
|
||||
formatGlobalDraftKey(expectedDraft),
|
||||
actualDraft,
|
||||
expectedDraft
|
||||
);
|
||||
if (fieldMismatch) {
|
||||
return fieldMismatch;
|
||||
}
|
||||
}
|
||||
|
||||
return `actual=${summarizeGlobalDrafts(actualDrafts)}; expected=${summarizeGlobalDrafts(expectedDrafts)}`;
|
||||
}
|
||||
|
||||
function describeGlobalDraftFieldMismatch(
|
||||
key: string,
|
||||
actual: GlobalDraft,
|
||||
expected: GlobalDraft
|
||||
): string | undefined {
|
||||
const fields: Array<"language" | "summary" | "value" | "isDraft"> = [
|
||||
"language",
|
||||
"summary",
|
||||
"value",
|
||||
"isDraft",
|
||||
];
|
||||
|
||||
for (const field of fields) {
|
||||
const actualValue = comparableGlobalDraftFieldValue(actual, field);
|
||||
const expectedValue = comparableGlobalDraftFieldValue(expected, field);
|
||||
if (JSON.stringify(actualValue) === JSON.stringify(expectedValue)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return `${key} ${field} differs: actual=${formatGlobalDraftFieldValue(
|
||||
actualValue
|
||||
)}; expected=${formatGlobalDraftFieldValue(expectedValue)}`;
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function comparableGlobalDraftFieldValue(
|
||||
draft: GlobalDraft,
|
||||
field: "language" | "summary" | "value" | "isDraft"
|
||||
): unknown {
|
||||
if (field === "summary" && typeof draft.summary === "string") {
|
||||
return normalizeText(draft.summary);
|
||||
}
|
||||
if (field === "value" && typeof draft.value === "string") {
|
||||
return normalizeText(draft.value);
|
||||
}
|
||||
if (field === "value") {
|
||||
return canonicalizeJsonValue(draft.value);
|
||||
}
|
||||
return draft[field];
|
||||
}
|
||||
|
||||
function formatGlobalDraftFieldValue(value: unknown): string {
|
||||
if (value === undefined) {
|
||||
return "(missing)";
|
||||
}
|
||||
return truncateForDetails(JSON.stringify(value), 300);
|
||||
}
|
||||
|
||||
function canonicalizeGlobalDrafts(drafts: GlobalDraft[]): unknown[] {
|
||||
return drafts
|
||||
.slice()
|
||||
.sort((left, right) => globalDraftSortKey(left).localeCompare(globalDraftSortKey(right)))
|
||||
.map((draft) =>
|
||||
canonicalizeJsonValue({
|
||||
type: draft.type,
|
||||
path: draft.path,
|
||||
triggerKind: draft.triggerKind,
|
||||
language: draft.language,
|
||||
summary:
|
||||
typeof draft.summary === "string" ? normalizeText(draft.summary) : draft.summary,
|
||||
value:
|
||||
typeof draft.value === "string"
|
||||
? normalizeText(draft.value)
|
||||
: canonicalizeJsonValue(draft.value),
|
||||
isDraft: draft.isDraft,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function globalDraftSortKey(draft: GlobalDraft): string {
|
||||
return `${draft.type}:${draft.triggerKind ?? ""}:${draft.path}`;
|
||||
}
|
||||
|
||||
function canonicalizeJsonValue(value: unknown): unknown {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(canonicalizeJsonValue);
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, nested]) => [key, canonicalizeJsonValue(nested)])
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function stringifyGlobalDraftValue(value: unknown): string {
|
||||
if (typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
return JSON.stringify(value ?? null, null, 2);
|
||||
}
|
||||
|
||||
function truncateForDetails(value: string, maxLength = 500): string {
|
||||
const normalized = value.replace(/\s+/g, " ").trim();
|
||||
if (normalized.length <= maxLength) {
|
||||
return normalized;
|
||||
}
|
||||
return `${normalized.slice(0, Math.max(0, maxLength - 3))}...`;
|
||||
}
|
||||
|
||||
function validateCliExpectations(
|
||||
assistantOutput: string,
|
||||
trace: CliTrace | undefined,
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { afterEach, describe, expect, it } from "bun:test";
|
||||
import { resolveWindmillBackendSettings } from "./windmillBackendSettings";
|
||||
|
||||
const ENV_KEYS = [
|
||||
"WMILL_AI_EVAL_BACKEND_URL",
|
||||
"WINDMILL_URL",
|
||||
"WINDMILL_BASE_URL",
|
||||
"REMOTE",
|
||||
"WMILL_AI_EVAL_BACKEND_EMAIL",
|
||||
"WMILL_AI_EVAL_BACKEND_PASSWORD",
|
||||
"WMILL_AI_EVAL_BACKEND_WORKSPACE",
|
||||
] as const;
|
||||
|
||||
const ORIGINAL_ENV = Object.fromEntries(
|
||||
ENV_KEYS.map((key) => [key, process.env[key]]),
|
||||
) as Record<(typeof ENV_KEYS)[number], string | undefined>;
|
||||
|
||||
afterEach(() => {
|
||||
for (const key of ENV_KEYS) {
|
||||
const value = ORIGINAL_ENV[key];
|
||||
if (value === undefined) {
|
||||
delete process.env[key];
|
||||
} else {
|
||||
process.env[key] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("resolveWindmillBackendSettings", () => {
|
||||
it("uses backend URL/auth defaults and the optional explicit workspace", () => {
|
||||
delete process.env.WMILL_AI_EVAL_BACKEND_URL;
|
||||
delete process.env.WINDMILL_URL;
|
||||
delete process.env.WINDMILL_BASE_URL;
|
||||
delete process.env.REMOTE;
|
||||
process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE = "shared-evals";
|
||||
|
||||
expect(resolveWindmillBackendSettings()).toEqual({
|
||||
baseUrl: "http://127.0.0.1:8000",
|
||||
email: "admin@windmill.dev",
|
||||
password: "changeme",
|
||||
workspaceOverride: "shared-evals",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not expose workspace retention knobs", () => {
|
||||
process.env.WMILL_AI_EVAL_BACKEND_URL = "http://backend.test/";
|
||||
|
||||
const settings = resolveWindmillBackendSettings();
|
||||
|
||||
expect(settings).toEqual({
|
||||
baseUrl: "http://backend.test",
|
||||
email: "admin@windmill.dev",
|
||||
password: "changeme",
|
||||
workspaceOverride: undefined,
|
||||
});
|
||||
expect(Object.keys(settings).sort()).toEqual([
|
||||
"baseUrl",
|
||||
"email",
|
||||
"password",
|
||||
"workspaceOverride",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -2,9 +2,7 @@ export interface WindmillBackendSettings {
|
||||
baseUrl: string;
|
||||
email: string;
|
||||
password: string;
|
||||
keepWorkspaces: boolean;
|
||||
workspaceOverride?: string;
|
||||
workspacePrefix: string;
|
||||
}
|
||||
|
||||
export function resolveWindmillBackendSettings(): WindmillBackendSettings {
|
||||
@@ -18,13 +16,9 @@ export function resolveWindmillBackendSettings(): WindmillBackendSettings {
|
||||
),
|
||||
email: process.env.WMILL_AI_EVAL_BACKEND_EMAIL ?? "admin@windmill.dev",
|
||||
password: process.env.WMILL_AI_EVAL_BACKEND_PASSWORD ?? "changeme",
|
||||
keepWorkspaces: isTruthy(process.env.WMILL_AI_EVAL_KEEP_WORKSPACES),
|
||||
workspaceOverride: sanitizeOptionalWorkspaceId(
|
||||
process.env.WMILL_AI_EVAL_BACKEND_WORKSPACE,
|
||||
),
|
||||
workspacePrefix: sanitizeWorkspacePrefix(
|
||||
process.env.WMILL_AI_EVAL_WORKSPACE_PREFIX ?? "ai-evals",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -43,25 +37,9 @@ function normalizeBaseUrl(value: string): string {
|
||||
return value.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function sanitizeWorkspacePrefix(value: string): string {
|
||||
const sanitized = value
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9-]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "");
|
||||
return sanitized.length > 0 ? sanitized : "ai-evals";
|
||||
}
|
||||
|
||||
function sanitizeOptionalWorkspaceId(
|
||||
value: string | undefined,
|
||||
): string | undefined {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function isTruthy(value: string | undefined): boolean {
|
||||
if (!value) {
|
||||
return false;
|
||||
}
|
||||
return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"workspace": {
|
||||
"scripts": [
|
||||
{
|
||||
"path": "f/evals/global/format_greeting",
|
||||
"summary": "Format a greeting for a provided name",
|
||||
"description": "Returns a plain greeting for the provided name.",
|
||||
"language": "bun",
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": ["name"]
|
||||
},
|
||||
"content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -5,15 +5,12 @@ import type { FrontendEvalModelConfig } from "../core/models";
|
||||
import { validateAppState, type AppFilesState } from "../core/validators";
|
||||
import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
|
||||
import { runAppEval } from "../adapters/frontend/core/app/appEvalRunner";
|
||||
import {
|
||||
DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
getFrontendApiKey,
|
||||
} from "./frontendCommon";
|
||||
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
|
||||
import { getFrontendApiKey } from "./frontendCommon";
|
||||
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
|
||||
|
||||
export function createAppModeRunner(
|
||||
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
transportSettings?: FrontendEvalTransportSettings,
|
||||
modelConfig: FrontendEvalModelConfig,
|
||||
backendSettings: WindmillBackendSettings,
|
||||
): ModeRunner<AppFilesState, AppFilesState, AppFilesState> {
|
||||
return {
|
||||
mode: "app",
|
||||
@@ -37,8 +34,7 @@ export function createAppModeRunner(
|
||||
appContext: context.evalCase?.runtime?.appContext,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
transport: transportSettings?.transport,
|
||||
backend: transportSettings?.backend,
|
||||
backend: backendSettings,
|
||||
runContext: context,
|
||||
},
|
||||
);
|
||||
|
||||
+6
-10
@@ -7,11 +7,8 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
|
||||
import { runFlowEval } from "../adapters/frontend/core/flow/flowEvalRunner";
|
||||
import type { FlowWorkspaceFixtures } from "../adapters/frontend/core/flow/fileHelpers";
|
||||
import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
|
||||
import {
|
||||
DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
getFrontendApiKey,
|
||||
} from "./frontendCommon";
|
||||
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
|
||||
import { getFrontendApiKey } from "./frontendCommon";
|
||||
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
|
||||
import {
|
||||
normalizeFlowInitialFixture,
|
||||
normalizeFlowStateFixture,
|
||||
@@ -19,9 +16,9 @@ import {
|
||||
} from "./flowFixtures";
|
||||
|
||||
export function createFlowModeRunner(
|
||||
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
backendValidation?: BackendValidationSettings,
|
||||
transportSettings?: FrontendEvalTransportSettings,
|
||||
modelConfig: FrontendEvalModelConfig,
|
||||
backendValidation: BackendValidationSettings | undefined,
|
||||
backendSettings: WindmillBackendSettings,
|
||||
): ModeRunner<FlowInitialFixture, FlowState, FlowState> {
|
||||
return {
|
||||
mode: "flow",
|
||||
@@ -49,8 +46,7 @@ export function createFlowModeRunner(
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
transport: transportSettings?.transport,
|
||||
backend: transportSettings?.backend,
|
||||
backend: backendSettings,
|
||||
runContext: context,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -1,12 +1,4 @@
|
||||
import {
|
||||
getFrontendEvalModel,
|
||||
resolveEvalModel,
|
||||
type FrontendEvalModelConfig,
|
||||
} from "../core/models";
|
||||
|
||||
export const DEFAULT_FRONTEND_EVAL_MODEL: FrontendEvalModelConfig = getFrontendEvalModel(
|
||||
resolveEvalModel("flow")
|
||||
);
|
||||
import type { FrontendEvalModelConfig } from "../core/models";
|
||||
|
||||
export function getFrontendApiKey(provider: FrontendEvalModelConfig["provider"]): string {
|
||||
const envName =
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { runGlobalEval } from "../adapters/frontend/core/global/globalEvalRunner";
|
||||
import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend";
|
||||
import type { FrontendEvalModelConfig } from "../core/models";
|
||||
import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types";
|
||||
import { validateGlobalState, type GlobalDraftState } from "../core/validators";
|
||||
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
|
||||
import { getFrontendApiKey } from "./frontendCommon";
|
||||
|
||||
export interface GlobalInitialFixture {
|
||||
workspace?: BenchmarkWorkspaceRunnables;
|
||||
}
|
||||
|
||||
export function createGlobalModeRunner(
|
||||
modelConfig: FrontendEvalModelConfig,
|
||||
backendSettings: WindmillBackendSettings,
|
||||
): ModeRunner<GlobalInitialFixture, GlobalDraftState, GlobalDraftState> {
|
||||
return {
|
||||
mode: "global",
|
||||
concurrency: 3,
|
||||
judgeThreshold: 80,
|
||||
async loadInitial(path) {
|
||||
return path ? await loadGlobalInitialFixture(path) : undefined;
|
||||
},
|
||||
async loadExpected(path) {
|
||||
return path ? await loadGlobalExpectedFixture(path) : undefined;
|
||||
},
|
||||
async run(prompt, initial, context) {
|
||||
const result = await runGlobalEval(
|
||||
prompt,
|
||||
getFrontendApiKey(modelConfig.provider),
|
||||
{
|
||||
workspaceFixtures: initial?.workspace,
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
backend: backendSettings,
|
||||
runContext: context,
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
success: result.success,
|
||||
actual: result.state,
|
||||
error: result.error,
|
||||
assistantMessageCount: result.assistantMessageCount,
|
||||
toolCallCount: result.toolCallCount,
|
||||
toolsUsed: result.toolsUsed,
|
||||
toolCallDetails: result.toolCallDetails,
|
||||
skillsInvoked: [],
|
||||
tokenUsage: result.tokenUsage,
|
||||
};
|
||||
},
|
||||
validate({ evalCase, actual, expected }) {
|
||||
return validateGlobalState({
|
||||
actual,
|
||||
expected,
|
||||
validate: evalCase.validate as GlobalValidationSpec | undefined,
|
||||
});
|
||||
},
|
||||
buildArtifacts(actual): BenchmarkArtifactFile[] {
|
||||
return [
|
||||
{
|
||||
path: "global-drafts.json",
|
||||
content: JSON.stringify(actual, null, 2) + "\n",
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function loadGlobalInitialFixture(path: string): Promise<GlobalInitialFixture> {
|
||||
const parsed = JSON.parse(await readFile(path, "utf8")) as GlobalInitialFixture;
|
||||
return {
|
||||
workspace: parsed.workspace ?? {},
|
||||
};
|
||||
}
|
||||
|
||||
async function loadGlobalExpectedFixture(path: string): Promise<GlobalDraftState> {
|
||||
return JSON.parse(await readFile(path, "utf8")) as GlobalDraftState;
|
||||
}
|
||||
@@ -6,16 +6,13 @@ import type { BenchmarkArtifactFile, ModeRunner } from "../core/types";
|
||||
import { BackendPreviewClient } from "../adapters/frontend/backendPreview";
|
||||
import { runScriptEval } from "../adapters/frontend/core/script/scriptEvalRunner";
|
||||
import type { ScriptEvalState } from "../adapters/frontend/core/script/fileHelpers";
|
||||
import {
|
||||
DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
getFrontendApiKey,
|
||||
} from "./frontendCommon";
|
||||
import type { FrontendEvalTransportSettings } from "../core/frontendTransport";
|
||||
import { getFrontendApiKey } from "./frontendCommon";
|
||||
import type { WindmillBackendSettings } from "../core/windmillBackendSettings";
|
||||
|
||||
export function createScriptModeRunner(
|
||||
modelConfig: FrontendEvalModelConfig = DEFAULT_FRONTEND_EVAL_MODEL,
|
||||
backendValidation?: BackendValidationSettings,
|
||||
transportSettings?: FrontendEvalTransportSettings,
|
||||
modelConfig: FrontendEvalModelConfig,
|
||||
backendValidation: BackendValidationSettings | undefined,
|
||||
backendSettings: WindmillBackendSettings,
|
||||
): ModeRunner<ScriptEvalState, ScriptEvalState, ScriptEvalState> {
|
||||
return {
|
||||
mode: "script",
|
||||
@@ -40,8 +37,7 @@ export function createScriptModeRunner(
|
||||
maxIterations: context.evalCase?.runtime?.maxTurns,
|
||||
provider: modelConfig.provider,
|
||||
model: modelConfig.model,
|
||||
transport: transportSettings?.transport,
|
||||
backend: transportSettings?.backend,
|
||||
backend: backendSettings,
|
||||
runContext: context,
|
||||
},
|
||||
);
|
||||
|
||||
+94
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled, NULL::text as workspace_id FROM password WHERE email = $1",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "email",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "login_type",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "super_admin",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "devops",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "verified",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "company",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "username",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "operator_only",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"name": "first_time_user",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"name": "role_source",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 11,
|
||||
"name": "disabled",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 12,
|
||||
"name": "workspace_id",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
null,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
null,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "0142d9dc9c1b57487dd5709a0376794f18d33e5bd6340c0189be7818cda64328"
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT email as \"email!\", login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, NULL::bool as operator_only, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id FROM password\n UNION ALL\n SELECT email as \"email!\", 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, true as operator_only, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC, \"email!\"\n LIMIT $1 OFFSET $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "email!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "login_type",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "verified!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "super_admin!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "devops!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "company",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "username",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "operator_only",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"name": "first_time_user!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"name": "role_source!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 11,
|
||||
"name": "disabled!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 12,
|
||||
"name": "workspace_id",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "16b4496c21d0619dab4521dca22e5fe144c59156a8f06d8592291684c49b2f37"
|
||||
}
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "exists",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "2a49e5b5486b650d96f3e9038cba8a5f2e75d3b12ee4718452e82c7318b1bcf4"
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT label, token_prefix, expiration, created_at, last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "label",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "token_prefix",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "expiration",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "last_used_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "scopes",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "workspace_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "read_only",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "52777947d60d5ddd6d28852a1cf104a5af2fd565706204b7d5d33594ff91e61e"
|
||||
}
|
||||
+1
-1
@@ -15,7 +15,7 @@
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO token\n (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id, read_only)\n SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10\n WHERE $9::varchar IS NULL OR NOT EXISTS(\n SELECT 1 FROM workspace WHERE id = $9 AND deleted = true\n )",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Timestamptz",
|
||||
"Bool",
|
||||
"TextArray",
|
||||
"Varchar",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "9a1bf7c424154172b56bbd29b51ef468969b5452150ee6fffe31d59b52a4f1c2"
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n UNION ALL\n SELECT email as \"email!\", true as operator_only, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n LIMIT $1 OFFSET $2",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "email!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "operator_only",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "login_type",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "verified!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "super_admin!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "devops!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "name",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "company",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "username",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 9,
|
||||
"name": "first_time_user!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 10,
|
||||
"name": "role_source!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 11,
|
||||
"name": "disabled!",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 12,
|
||||
"name": "workspace_id",
|
||||
"type_info": "Text"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "c17c39add3f70218dbae38595a909d02cfabe7e0864af577df6968428ac448ef"
|
||||
}
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT label, token_prefix, expiration, created_at, last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)\n ORDER BY created_at DESC LIMIT $2 OFFSET $3",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "label",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "token_prefix",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "expiration",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "created_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "last_used_at",
|
||||
"type_info": "Timestamptz"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "scopes",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "workspace_id",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "read_only",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int8",
|
||||
"Int8"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "e001fb68c60fa736cecce52be49c9fb0714c9c91a7ea3924811d345bee94c013"
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "UPDATE token SET last_used_at = now() WHERE\n token_hash = $1\n AND (expiration > NOW() OR expiration IS NULL)\n AND (workspace_id IS NULL OR workspace_id = $2)\n RETURNING owner, email, super_admin, scopes, label, read_only",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "owner",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "email",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "super_admin",
|
||||
"type_info": "Bool"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "scopes",
|
||||
"type_info": "TextArray"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "label",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "read_only",
|
||||
"type_info": "Bool"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "ed82ffc4d806a19519701e39abc43e8e250cc9772cca4a7dae3442be86f4060c"
|
||||
}
|
||||
Generated
+296
-282
File diff suppressed because it is too large
Load Diff
+2
-2
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "windmill"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
authors.workspace = true
|
||||
edition.workspace = true
|
||||
|
||||
@@ -87,7 +87,7 @@ members = [
|
||||
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
edition = "2021"
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
c6cd1afe2d9e04809b30751cd1687b28a65e62b1
|
||||
19a76a09ffb43649ee19e62d07e8b8a42d78757b
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE token DROP COLUMN IF EXISTS read_only;
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Add a flag to restrict a token to read-only HTTP endpoints.
|
||||
-- Orthogonal to `scopes`: even if scopes grant write/run, this flag denies
|
||||
-- mutating methods (POST/PUT/PATCH/DELETE) and Run actions.
|
||||
ALTER TABLE token ADD COLUMN read_only BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -0,0 +1,3 @@
|
||||
-- No-op: clearing a stray `auto_kind = 'lib'` value on failure/trigger/approval
|
||||
-- scripts is not reversible (the original NULL/'lib' distinction is lost), and
|
||||
-- restoring `'lib'` here would re-hide these scripts from their pickers.
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Failure, Trigger, and Approval scripts are runnable entrypoints by
|
||||
-- definition. A prior parser regression occasionally classified them as
|
||||
-- `auto_kind = 'lib'`, which hid them from the flow error-handler /
|
||||
-- trigger / approval pickers. Clear those stray values so existing
|
||||
-- affected scripts re-appear without requiring a redeploy.
|
||||
UPDATE script
|
||||
SET auto_kind = NULL
|
||||
WHERE auto_kind = 'lib'
|
||||
AND kind IN ('failure', 'trigger', 'approval');
|
||||
+24
-24
@@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windmill-common"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"anyhow",
|
||||
@@ -6263,7 +6263,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-macros"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -6275,7 +6275,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"convert_case",
|
||||
"serde",
|
||||
@@ -6284,7 +6284,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-bash"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -6296,7 +6296,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-csharp"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -6308,7 +6308,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-go"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"gosyn",
|
||||
@@ -6320,7 +6320,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-graphql"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -6332,7 +6332,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-java"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -6344,7 +6344,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-nu"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"nu-parser",
|
||||
@@ -6355,7 +6355,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-php"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -6366,7 +6366,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.14.0",
|
||||
@@ -6378,7 +6378,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py-asset"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"rustpython-ast",
|
||||
@@ -6389,7 +6389,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-py-imports"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-recursion",
|
||||
@@ -6411,7 +6411,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-r"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde_json",
|
||||
@@ -6423,7 +6423,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ruby"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -6437,7 +6437,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-rust"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"convert_case",
|
||||
@@ -6454,7 +6454,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-sql"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -6467,7 +6467,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-sql-asset"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde",
|
||||
@@ -6479,7 +6479,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ts"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"lazy_static",
|
||||
@@ -6497,7 +6497,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-ts-asset"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde-wasm-bindgen",
|
||||
@@ -6513,7 +6513,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-wac"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"rustpython-ast",
|
||||
@@ -6529,7 +6529,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-wasm"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"getrandom 0.2.17",
|
||||
@@ -6561,7 +6561,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-parser-yaml"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"serde",
|
||||
@@ -6572,7 +6572,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "windmill-types"
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bitflags",
|
||||
|
||||
@@ -12,7 +12,7 @@ resolver = "2"
|
||||
members = ["."]
|
||||
|
||||
[workspace.package]
|
||||
version = "1.699.0"
|
||||
version = "1.702.1"
|
||||
edition = "2021"
|
||||
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
|
||||
|
||||
|
||||
+81
-54
@@ -6,6 +6,8 @@ use windmill_common::{
|
||||
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
|
||||
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
|
||||
pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
|
||||
#[cfg(feature = "operator")]
|
||||
pub const DEFAULT_MAX_CONNECTIONS_OPERATOR: u32 = 2;
|
||||
|
||||
pub async fn initial_connection() -> Result<sqlx::Pool<sqlx::Postgres>, error::Error> {
|
||||
let connect_options = get_database_url().await?.connect_options().await?;
|
||||
@@ -16,12 +18,35 @@ pub async fn initial_connection() -> Result<sqlx::Pool<sqlx::Postgres>, error::E
|
||||
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))
|
||||
}
|
||||
|
||||
/// Connect to the database for the Kubernetes operator process.
|
||||
///
|
||||
/// Long-running operator pods need IAM RDS / Entra ID token refresh just like the server,
|
||||
/// otherwise new pool connections start failing once the initial token expires (~15 min).
|
||||
#[cfg(feature = "operator")]
|
||||
pub async fn operator_connection(
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
|
||||
let database_url = get_database_url().await?;
|
||||
let pool = connect(
|
||||
database_url.clone(),
|
||||
DEFAULT_MAX_CONNECTIONS_OPERATOR,
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
spawn_token_refresh_task(pool.clone(), database_url, killpill_rx);
|
||||
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
pub async fn connect_db(
|
||||
server_mode: bool,
|
||||
indexer_mode: bool,
|
||||
worker_mode: bool,
|
||||
num_workers: i32,
|
||||
#[cfg(feature = "private")] mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
#[cfg(feature = "private")] killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) -> anyhow::Result<sqlx::Pool<sqlx::Postgres>> {
|
||||
use anyhow::Context;
|
||||
|
||||
@@ -43,70 +68,72 @@ pub async fn connect_db(
|
||||
let pool = connect(database_url.clone(), max_connections, worker_mode).await?;
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
{
|
||||
let needs_token_refresh = matches!(
|
||||
database_url,
|
||||
DatabaseUrl::IamRds(_) | DatabaseUrl::EntraId(_)
|
||||
);
|
||||
let label = match &database_url {
|
||||
DatabaseUrl::IamRds(_) => "IAM RDS",
|
||||
DatabaseUrl::EntraId(_) => "Entra ID",
|
||||
DatabaseUrl::Static(_) => "",
|
||||
};
|
||||
if needs_token_refresh {
|
||||
let pool2 = pool.clone();
|
||||
let database_url2 = database_url.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = killpill_rx.recv() => {
|
||||
break;
|
||||
}
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
|
||||
if !database_url2.needs_refresh().await {
|
||||
continue;
|
||||
}
|
||||
let new_url = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
get_database_url(),
|
||||
)
|
||||
.await;
|
||||
match new_url {
|
||||
Ok(Ok(new_url)) => {
|
||||
match new_url.connect_options().await {
|
||||
Ok(connect_options) => {
|
||||
pool2.set_connect_options(connect_options);
|
||||
tracing::info!("Refreshed {label} URL successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"Error getting {label} connect options, retrying in 10s: {e}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!(
|
||||
"Error refreshing {label} URL, trying again in 10s: {e}"
|
||||
);
|
||||
continue;
|
||||
spawn_token_refresh_task(pool.clone(), database_url, killpill_rx);
|
||||
|
||||
Ok(pool)
|
||||
}
|
||||
|
||||
/// Spawn a background task that refreshes IAM RDS / Entra ID tokens before they expire
|
||||
/// and updates the pool's connect options so new connections use the fresh token.
|
||||
/// No-op for static (password-based) database URLs.
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
pub fn spawn_token_refresh_task(
|
||||
pool: sqlx::Pool<sqlx::Postgres>,
|
||||
database_url: DatabaseUrl,
|
||||
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
|
||||
) {
|
||||
let label = match &database_url {
|
||||
DatabaseUrl::IamRds(_) => "IAM RDS",
|
||||
DatabaseUrl::EntraId(_) => "Entra ID",
|
||||
DatabaseUrl::Static(_) => return,
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = killpill_rx.recv() => {
|
||||
break;
|
||||
}
|
||||
_ = tokio::time::sleep(std::time::Duration::from_secs(10)) => {
|
||||
if !database_url.needs_refresh().await {
|
||||
continue;
|
||||
}
|
||||
let new_url = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
get_database_url(),
|
||||
)
|
||||
.await;
|
||||
match new_url {
|
||||
Ok(Ok(new_url)) => {
|
||||
match new_url.connect_options().await {
|
||||
Ok(connect_options) => {
|
||||
pool.set_connect_options(connect_options);
|
||||
tracing::info!("Refreshed {label} URL successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"Timeout after 10s refreshing {label} URL, trying again in 10s: {e}"
|
||||
"Error getting {label} connect options, retrying in 10s: {e}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!(
|
||||
"Error refreshing {label} URL, trying again in 10s: {e}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"Timeout after 10s refreshing {label} URL, trying again in 10s: {e}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(pool)
|
||||
});
|
||||
}
|
||||
|
||||
pub async fn connect(
|
||||
|
||||
+18
-1
@@ -670,7 +670,24 @@ async fn windmill_main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt::init();
|
||||
tracing::info!("Starting Windmill Kubernetes operator...");
|
||||
tracing::info!("Connecting to database...");
|
||||
let db = crate::db_connect::initial_connection().await?;
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
let (operator_killpill_tx, operator_killpill_rx) =
|
||||
tokio::sync::broadcast::channel::<()>(2);
|
||||
|
||||
let db = crate::db_connect::operator_connection(
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
operator_killpill_rx,
|
||||
)
|
||||
.await?;
|
||||
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
tokio::spawn(async move {
|
||||
if let Ok(()) = tokio::signal::ctrl_c().await {
|
||||
let _ = operator_killpill_tx.send(());
|
||||
}
|
||||
});
|
||||
|
||||
tracing::info!("Database connected. Starting ConfigMap watcher...");
|
||||
windmill_operator::run(db).await?;
|
||||
return Ok(());
|
||||
|
||||
@@ -886,11 +886,13 @@ pub async fn reload_otel_tracing_proxy_setting(conn: &Connection) {
|
||||
let mut current = OTEL_TRACING_PROXY_SETTINGS.write().await;
|
||||
if current.enabled != new_settings.enabled
|
||||
|| current.enabled_languages != new_settings.enabled_languages
|
||||
|| current.no_proxy_hosts != new_settings.no_proxy_hosts
|
||||
{
|
||||
tracing::info!(
|
||||
"OTEL tracing proxy settings changed: enabled={}, languages={:?}",
|
||||
"OTEL tracing proxy settings changed: enabled={}, languages={:?}, no_proxy_hosts={:?}",
|
||||
new_settings.enabled,
|
||||
new_settings.enabled_languages
|
||||
new_settings.enabled_languages,
|
||||
new_settings.no_proxy_hosts,
|
||||
);
|
||||
*current = new_settings;
|
||||
}
|
||||
|
||||
@@ -577,6 +577,63 @@ export function main() {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression test: a `//nobundling` script that pulls a package whose CJS
|
||||
/// internals do bare-specifier `require()` of a sibling dependency.
|
||||
///
|
||||
/// Before the `--preserve-symlinks` fix, Bun 1.2/1.3+ would follow the
|
||||
/// directory symlink in `node_modules/@langchain/core` to its global cache
|
||||
/// entry, walk parent dirs from the cache realpath, and fail to find
|
||||
/// `node_modules/zod` — producing:
|
||||
/// ENOENT while resolving package 'zod/v3' from
|
||||
/// '.../cache_nomount/bun/@langchain/core@<ver>@@@1/dist/runnables/base.js'
|
||||
///
|
||||
/// The fix passes `--preserve-symlinks` so Bun resolves from the
|
||||
/// symlink path under `<job_dir>/node_modules/`, where `zod` is a sibling.
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_bun_nobundling_transitive_require(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let content = r#"//nobundling
|
||||
import { ChatPromptTemplate } from "@langchain/core/prompts";
|
||||
|
||||
export async function main() {
|
||||
const tpl = ChatPromptTemplate.fromMessages([
|
||||
["system", "you are a {role}"],
|
||||
["human", "{input}"],
|
||||
]);
|
||||
const out = await tpl.formatMessages({ role: "tester", input: "ping" });
|
||||
return out.length;
|
||||
}
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
modules: None,
|
||||
tag: None,
|
||||
});
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, false, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result, serde_json::json!(2));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Native Mode Tests (requires deno_core feature)
|
||||
// ============================================================================
|
||||
|
||||
@@ -760,6 +760,112 @@ def main():
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "python")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_python_result_preserves_infinity_in_string(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let content = r#"
|
||||
def main():
|
||||
return {
|
||||
"plain": "Infinity",
|
||||
"embedded": "value=-Infinity end",
|
||||
"nan_word": "this is NaN inside text",
|
||||
"nested": [{"k": "Infinity"}],
|
||||
}
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
language: ScriptLang::Python3,
|
||||
lock: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
modules: None,
|
||||
tag: None,
|
||||
});
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, false, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
serde_json::json!({
|
||||
"plain": "Infinity",
|
||||
"embedded": "value=-Infinity end",
|
||||
"nan_word": "this is NaN inside text",
|
||||
"nested": [{"k": "Infinity"}],
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "python")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_python_result_non_finite_floats_become_null(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
let server = ApiServer::start(db.clone()).await?;
|
||||
let port = server.addr.port();
|
||||
|
||||
let content = r#"
|
||||
def main():
|
||||
return {
|
||||
"inf": float("inf"),
|
||||
"neg_inf": float("-inf"),
|
||||
"nan": float("nan"),
|
||||
"finite": 1.5,
|
||||
"nested": [float("inf"), {"x": float("nan")}],
|
||||
}
|
||||
"#
|
||||
.to_owned();
|
||||
|
||||
let job = JobPayload::Code(RawCode {
|
||||
hash: None,
|
||||
content,
|
||||
path: None,
|
||||
language: ScriptLang::Python3,
|
||||
lock: None,
|
||||
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
|
||||
.into(),
|
||||
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
|
||||
cache_ttl: None,
|
||||
cache_ignore_s3_path: None,
|
||||
dedicated_worker: None,
|
||||
modules: None,
|
||||
tag: None,
|
||||
});
|
||||
|
||||
let result = run_job_in_new_worker_until_complete(&db, false, job, port)
|
||||
.await
|
||||
.json_result()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
result,
|
||||
serde_json::json!({
|
||||
"inf": null,
|
||||
"neg_inf": null,
|
||||
"nan": null,
|
||||
"finite": 1.5,
|
||||
"nested": [null, {"x": null}],
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "python")]
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn test_python_global_site_packages(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use sqlx::{Pool, Postgres};
|
||||
use windmill_api_client::types::{NewScript, ScriptLang};
|
||||
use windmill_test_utils::init_client;
|
||||
|
||||
fn quick_ns(content: &str, path: &str, kind: Option<&str>) -> NewScript {
|
||||
NewScript {
|
||||
content: content.into(),
|
||||
language: ScriptLang::Bun,
|
||||
lock: None,
|
||||
parent_hash: None,
|
||||
path: path.into(),
|
||||
concurrent_limit: None,
|
||||
concurrency_time_window_s: None,
|
||||
cache_ttl: None,
|
||||
dedicated_worker: None,
|
||||
description: "".to_string(),
|
||||
draft_only: None,
|
||||
envs: vec![],
|
||||
is_template: None,
|
||||
kind: kind.map(|s| s.to_string()),
|
||||
summary: "".to_string(),
|
||||
tag: None,
|
||||
schema: HashMap::new(),
|
||||
ws_error_handler_muted: Some(false),
|
||||
priority: None,
|
||||
delete_after_secs: None,
|
||||
timeout: None,
|
||||
restart_unless_cancelled: None,
|
||||
deployment_message: None,
|
||||
concurrency_key: None,
|
||||
visible_to_runner_only: None,
|
||||
auto_kind: None,
|
||||
codebase: None,
|
||||
has_preprocessor: None,
|
||||
on_behalf_of_email: None,
|
||||
assets: vec![],
|
||||
modules: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Regression: a `failure`-kind script must never be marked `auto_kind = 'lib'`
|
||||
/// even if the parser fails to detect a `main` function, because the flow
|
||||
/// error-handler picker filters out lib scripts and would otherwise hide it.
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn failure_kind_script_without_main_is_not_marked_lib(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
let (client, _port, _s) = init_client(db.clone()).await;
|
||||
|
||||
// Content with no `main` — TS parser would normally set auto_kind = 'lib'.
|
||||
client
|
||||
.create_script(
|
||||
"test-workspace",
|
||||
&quick_ns(
|
||||
"export function notMain() { return 42 }",
|
||||
"u/test-user/failure_no_main",
|
||||
Some("failure"),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let auto_kind: Option<String> = sqlx::query_scalar(
|
||||
"SELECT auto_kind FROM script \
|
||||
WHERE workspace_id = $1 AND path = $2",
|
||||
)
|
||||
.bind("test-workspace")
|
||||
.bind("u/test-user/failure_no_main")
|
||||
.fetch_one(&db)
|
||||
.await?;
|
||||
|
||||
assert_ne!(
|
||||
auto_kind.as_deref(),
|
||||
Some("lib"),
|
||||
"failure-kind script must not be marked as 'lib' auto_kind, got {:?}",
|
||||
auto_kind
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sibling: a normal `script` kind WITHOUT main should still be marked `lib`
|
||||
/// (so it stays hidden from the regular script picker). Guards against an
|
||||
/// over-broad sanitizer accidentally clearing the value for plain scripts.
|
||||
#[sqlx::test(fixtures("base"))]
|
||||
async fn regular_script_without_main_is_still_marked_lib(
|
||||
db: Pool<Postgres>,
|
||||
) -> anyhow::Result<()> {
|
||||
let (client, _port, _s) = init_client(db.clone()).await;
|
||||
|
||||
client
|
||||
.create_script(
|
||||
"test-workspace",
|
||||
&quick_ns(
|
||||
"export function notMain() { return 42 }",
|
||||
"u/test-user/script_no_main",
|
||||
Some("script"),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let auto_kind: Option<String> = sqlx::query_scalar(
|
||||
"SELECT auto_kind FROM script \
|
||||
WHERE workspace_id = $1 AND path = $2",
|
||||
)
|
||||
.bind("test-workspace")
|
||||
.bind("u/test-user/script_no_main")
|
||||
.fetch_one(&db)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
auto_kind.as_deref(),
|
||||
Some("lib"),
|
||||
"regular script without main should be marked 'lib', got {:?}",
|
||||
auto_kind
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -174,6 +174,7 @@ fn make_authed() -> windmill_api_auth::ApiAuthed {
|
||||
scopes: None,
|
||||
username_override: None,
|
||||
token_prefix: None,
|
||||
read_only: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -326,8 +326,10 @@ pub fn json_to_document(value: serde_json::Value) -> aws_smithy_types::Document
|
||||
}
|
||||
Value::Array(arr) => Document::Array(arr.into_iter().map(json_to_document).collect()),
|
||||
Value::Number(num) => {
|
||||
if let Some(i) = num.as_i64() {
|
||||
Document::Number(aws_smithy_types::Number::PosInt(i as u64))
|
||||
if let Some(u) = num.as_u64() {
|
||||
Document::Number(aws_smithy_types::Number::PosInt(u))
|
||||
} else if let Some(i) = num.as_i64() {
|
||||
Document::Number(aws_smithy_types::Number::NegInt(i))
|
||||
} else if let Some(f) = num.as_f64() {
|
||||
Document::Number(aws_smithy_types::Number::Float(f))
|
||||
} else {
|
||||
@@ -844,6 +846,36 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_to_document_preserves_negative_integers() {
|
||||
let value = serde_json::json!(-1);
|
||||
let doc = json_to_document(value);
|
||||
assert!(matches!(
|
||||
doc,
|
||||
aws_smithy_types::Document::Number(aws_smithy_types::Number::NegInt(-1))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_to_document_handles_large_u64_above_i64_max() {
|
||||
let value = serde_json::json!(u64::MAX);
|
||||
let doc = json_to_document(value);
|
||||
assert!(matches!(
|
||||
doc,
|
||||
aws_smithy_types::Document::Number(aws_smithy_types::Number::PosInt(u)) if u == u64::MAX
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_to_document_handles_positive_integers() {
|
||||
let value = serde_json::json!(42);
|
||||
let doc = json_to_document(value);
|
||||
assert!(matches!(
|
||||
doc,
|
||||
aws_smithy_types::Document::Number(aws_smithy_types::Number::PosInt(42))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn openai_messages_to_bedrock_adds_cache_points_when_enabled() {
|
||||
let messages = vec![
|
||||
|
||||
@@ -5,6 +5,7 @@ pub mod ai_google;
|
||||
pub mod ai_providers;
|
||||
pub mod ai_types;
|
||||
pub mod image_handler;
|
||||
pub mod providers;
|
||||
pub mod query_builder;
|
||||
pub mod sse;
|
||||
pub mod types;
|
||||
|
||||
+4
-4
@@ -1,7 +1,4 @@
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use windmill_ai::{
|
||||
use crate::{
|
||||
ai_google::parse_data_url,
|
||||
ai_providers::AIProvider,
|
||||
image_handler::prepare_messages_for_api,
|
||||
@@ -10,6 +7,9 @@ use windmill_ai::{
|
||||
types::*,
|
||||
utils::{extract_text_content, should_use_structured_output_tool},
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
|
||||
/// Anthropic API version for standard API
|
||||
+6
-6
@@ -1,4 +1,4 @@
|
||||
//! AWS Bedrock provider for the AI agent.
|
||||
//! AWS Bedrock provider for AI requests.
|
||||
//!
|
||||
//! Uses shared SDK code from windmill_ai::ai_bedrock for:
|
||||
//! - BedrockClient (SDK wrapper with auth)
|
||||
@@ -6,16 +6,16 @@
|
||||
//! - Stream event parsing
|
||||
//! - Helper utilities
|
||||
|
||||
use std::collections::HashMap;
|
||||
use windmill_ai::{
|
||||
use crate::{
|
||||
image_handler::prepare_messages_for_api,
|
||||
query_builder::{ParsedResponse, StreamEventSink},
|
||||
types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef},
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
|
||||
// Import shared Bedrock helpers for worker-specific orchestration.
|
||||
use windmill_ai::ai_bedrock::{
|
||||
// Import shared Bedrock helpers for provider orchestration.
|
||||
use crate::ai_bedrock::{
|
||||
bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop,
|
||||
bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta,
|
||||
bedrock_stream_event_to_tool_start, build_tool_config, create_inference_config,
|
||||
@@ -24,7 +24,7 @@ use windmill_ai::ai_bedrock::{
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Query Builder (Worker-specific orchestration)
|
||||
// Query Builder
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Default)]
|
||||
+2
-2
@@ -1,5 +1,4 @@
|
||||
use async_trait::async_trait;
|
||||
use windmill_ai::{
|
||||
use crate::{
|
||||
ai_google::{
|
||||
openai_messages_to_gemini, openai_tools_to_gemini, GeminiGenerationConfig,
|
||||
GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart,
|
||||
@@ -10,6 +9,7 @@ use windmill_ai::{
|
||||
sse::{GeminiSSEParser, SSEParser},
|
||||
types::*,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
|
||||
// ============================================================================
|
||||
@@ -0,0 +1,31 @@
|
||||
pub mod anthropic;
|
||||
#[cfg(feature = "bedrock")]
|
||||
pub mod bedrock;
|
||||
pub mod google_ai;
|
||||
pub mod openai;
|
||||
pub mod openrouter;
|
||||
pub mod other;
|
||||
|
||||
use crate::{ai_providers::AIProvider, query_builder::QueryBuilder, types::ProviderWithResource};
|
||||
|
||||
use self::{
|
||||
anthropic::AnthropicQueryBuilder, google_ai::GoogleAIQueryBuilder, openai::OpenAIQueryBuilder,
|
||||
openrouter::OpenRouterQueryBuilder, other::OtherQueryBuilder,
|
||||
};
|
||||
|
||||
/// Factory function to create the appropriate query builder for a provider.
|
||||
pub fn create_query_builder(provider: &ProviderWithResource) -> Box<dyn QueryBuilder> {
|
||||
match provider.kind {
|
||||
AIProvider::GoogleAI => {
|
||||
Box::new(GoogleAIQueryBuilder::new(provider.get_platform().clone()))
|
||||
}
|
||||
AIProvider::OpenAI => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())),
|
||||
AIProvider::Anthropic => Box::new(AnthropicQueryBuilder::new(
|
||||
provider.kind.clone(),
|
||||
provider.get_platform().clone(),
|
||||
provider.get_enable_1m_context(),
|
||||
)),
|
||||
AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()),
|
||||
_ => Box::new(OtherQueryBuilder::new(provider.kind.clone())),
|
||||
}
|
||||
}
|
||||
+4
-4
@@ -1,7 +1,4 @@
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use windmill_ai::{
|
||||
use crate::{
|
||||
ai_providers::AIProvider,
|
||||
ai_types::OpenAIToolCall,
|
||||
image_handler::{prepare_messages_for_api, s3_object_to_content_part},
|
||||
@@ -10,6 +7,9 @@ use windmill_ai::{
|
||||
types::*,
|
||||
utils::extract_text_content,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::value::RawValue;
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
|
||||
// Responses API structures
|
||||
+5
-5
@@ -1,15 +1,15 @@
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json;
|
||||
use windmill_ai::{
|
||||
use crate::{
|
||||
ai_providers::AIProvider,
|
||||
image_handler::prepare_messages_for_api,
|
||||
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink},
|
||||
types::*,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json;
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
|
||||
use crate::ai::providers::other::OtherQueryBuilder;
|
||||
use crate::providers::other::OtherQueryBuilder;
|
||||
|
||||
// OpenRouter-specific types
|
||||
#[derive(Serialize)]
|
||||
+4
-4
@@ -1,7 +1,4 @@
|
||||
use async_trait::async_trait;
|
||||
use serde::Serialize;
|
||||
use serde_json;
|
||||
use windmill_ai::{
|
||||
use crate::{
|
||||
ai_providers::AIProvider,
|
||||
image_handler::prepare_messages_for_api,
|
||||
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink},
|
||||
@@ -9,6 +6,9 @@ use windmill_ai::{
|
||||
types::*,
|
||||
utils::should_use_structured_output_tool,
|
||||
};
|
||||
use async_trait::async_trait;
|
||||
use serde::Serialize;
|
||||
use serde_json;
|
||||
use windmill_common::{client::AuthedClient, error::Error};
|
||||
|
||||
#[derive(Serialize, Debug, Clone)]
|
||||
@@ -194,6 +194,7 @@ impl AuthCache {
|
||||
scopes: None,
|
||||
username_override,
|
||||
token_prefix: claims.audit_span,
|
||||
read_only: false,
|
||||
};
|
||||
let job_id = claims.job_id.and_then(|j| uuid::Uuid::from_str(&j).ok());
|
||||
AUTH_CACHE.insert(
|
||||
@@ -221,11 +222,20 @@ impl AuthCache {
|
||||
token_hash = $1
|
||||
AND (expiration > NOW() OR expiration IS NULL)
|
||||
AND (workspace_id IS NULL OR workspace_id = $2)
|
||||
RETURNING owner, email, super_admin, scopes, label",
|
||||
RETURNING owner, email, super_admin, scopes, label, read_only",
|
||||
t_hash,
|
||||
w_id.as_ref(),
|
||||
)
|
||||
.map(|x| (x.owner, x.email, x.super_admin, x.scopes, x.label))
|
||||
.map(|x| {
|
||||
(
|
||||
x.owner,
|
||||
x.email,
|
||||
x.super_admin,
|
||||
x.scopes,
|
||||
x.label,
|
||||
x.read_only,
|
||||
)
|
||||
})
|
||||
.fetch_optional(&self.db)
|
||||
.await
|
||||
.ok()
|
||||
@@ -234,7 +244,9 @@ impl AuthCache {
|
||||
if let Some(user) = user_o {
|
||||
let authed_o = {
|
||||
match user {
|
||||
(Some(owner), Some(email), super_admin, _, label) if w_id.is_some() => {
|
||||
(Some(owner), Some(email), super_admin, _, label, read_only)
|
||||
if w_id.is_some() =>
|
||||
{
|
||||
let username_override = username_override_from_label(label);
|
||||
if let Some((prefix, name)) = owner.split_once('/') {
|
||||
if prefix == "u" {
|
||||
@@ -280,6 +292,7 @@ impl AuthCache {
|
||||
scopes: None,
|
||||
username_override,
|
||||
token_prefix: Some(safe_token_prefix(token)),
|
||||
read_only,
|
||||
})
|
||||
} else {
|
||||
let groups = vec![name.to_string()];
|
||||
@@ -305,6 +318,7 @@ impl AuthCache {
|
||||
scopes: None,
|
||||
username_override,
|
||||
token_prefix: Some(safe_token_prefix(token)),
|
||||
read_only,
|
||||
})
|
||||
}
|
||||
} else {
|
||||
@@ -320,10 +334,11 @@ impl AuthCache {
|
||||
scopes: None,
|
||||
username_override,
|
||||
token_prefix: Some(safe_token_prefix(token)),
|
||||
read_only,
|
||||
})
|
||||
}
|
||||
}
|
||||
(_, Some(email), super_admin, scopes, label) => {
|
||||
(_, Some(email), super_admin, scopes, label, read_only) => {
|
||||
let username_override = username_override_from_label(label);
|
||||
if w_id.is_some() {
|
||||
let row_o = sqlx::query!(
|
||||
@@ -368,6 +383,7 @@ impl AuthCache {
|
||||
scopes,
|
||||
username_override,
|
||||
token_prefix: Some(safe_token_prefix(token)),
|
||||
read_only,
|
||||
})
|
||||
}
|
||||
None if super_admin => Some(ApiAuthed {
|
||||
@@ -380,6 +396,7 @@ impl AuthCache {
|
||||
scopes,
|
||||
username_override,
|
||||
token_prefix: Some(safe_token_prefix(token)),
|
||||
read_only,
|
||||
}),
|
||||
None => None,
|
||||
}
|
||||
@@ -394,6 +411,7 @@ impl AuthCache {
|
||||
scopes,
|
||||
username_override,
|
||||
token_prefix: Some(safe_token_prefix(token)),
|
||||
read_only,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -428,6 +446,7 @@ impl AuthCache {
|
||||
scopes: None,
|
||||
username_override: None,
|
||||
token_prefix: Some(safe_token_prefix(token)),
|
||||
read_only: false,
|
||||
};
|
||||
Some(OptJobAuthed { authed, job_id: None })
|
||||
} else {
|
||||
@@ -630,6 +649,7 @@ pub async fn resolve_opt_job_authed(
|
||||
scopes: None,
|
||||
username_override: None,
|
||||
token_prefix: None,
|
||||
read_only: false,
|
||||
};
|
||||
return Ok((OptJobAuthed { authed, job_id: None }, parts));
|
||||
}
|
||||
@@ -667,12 +687,11 @@ pub async fn resolve_opt_job_authed(
|
||||
cache.get_opt_job_authed(workspace_id.clone(), &token).await
|
||||
{
|
||||
let authed = &mut opt_job_authed.authed;
|
||||
let path = original_uri.path();
|
||||
let method = parts.method.as_str();
|
||||
if authed.scopes.is_some() {
|
||||
transform_old_scope_to_new_scope(authed.scopes.as_mut());
|
||||
|
||||
let path = original_uri.path();
|
||||
let method = parts.method.as_str();
|
||||
|
||||
if let Err(err) = crate::scopes::check_scopes_for_route(
|
||||
authed.scopes.as_deref(),
|
||||
path,
|
||||
@@ -681,6 +700,27 @@ pub async fn resolve_opt_job_authed(
|
||||
return Err((err, parts));
|
||||
}
|
||||
}
|
||||
if authed.read_only {
|
||||
// MCP transport runs over POST (streamable HTTP / SSE handshake),
|
||||
// so the middleware can't safely reject mutating methods on it —
|
||||
// the MCP runner itself filters out write tools and rejects
|
||||
// mutating tool calls for read-only tokens. Narrow to the actual
|
||||
// transport endpoints: anything else under `/api/mcp/*` (OAuth
|
||||
// approve, token exchange, client registration) must still go
|
||||
// through the read-only check, otherwise a read-only token
|
||||
// could approve an OAuth flow that mints a new non-read-only
|
||||
// token.
|
||||
let is_mcp_transport = path == "/api/mcp/gateway"
|
||||
|| (path.starts_with("/api/mcp/w/")
|
||||
&& (path.ends_with("/mcp")
|
||||
|| path.ends_with("/sse")
|
||||
|| path.ends_with("/list_tools")));
|
||||
if !is_mcp_transport {
|
||||
if let Err(err) = crate::scopes::check_read_only_for_route(path, method) {
|
||||
return Err((err, parts));
|
||||
}
|
||||
}
|
||||
}
|
||||
parts.extensions.insert(authed.clone());
|
||||
|
||||
Span::current().record("username", &authed.username.as_str());
|
||||
|
||||
@@ -55,6 +55,7 @@ pub struct ApiAuthed {
|
||||
pub scopes: Option<Vec<String>>,
|
||||
pub username_override: Option<String>,
|
||||
pub token_prefix: Option<String>,
|
||||
pub read_only: bool,
|
||||
}
|
||||
|
||||
impl ApiAuthed {
|
||||
@@ -103,6 +104,7 @@ impl From<Authed> for ApiAuthed {
|
||||
scopes: value.scopes,
|
||||
username_override: None,
|
||||
token_prefix: value.token_prefix,
|
||||
read_only: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -183,6 +185,10 @@ impl windmill_mcp::server::McpAuth for ApiAuthed {
|
||||
fn scopes(&self) -> Option<&[String]> {
|
||||
self.scopes.as_deref()
|
||||
}
|
||||
|
||||
fn read_only(&self) -> bool {
|
||||
self.read_only
|
||||
}
|
||||
}
|
||||
|
||||
// ------------ Utility functions ------------
|
||||
@@ -478,6 +484,7 @@ pub async fn fetch_api_authed_from_permissioned_as(
|
||||
scopes: authed.scopes,
|
||||
username_override: None,
|
||||
token_prefix: authed.token_prefix,
|
||||
read_only: false,
|
||||
};
|
||||
|
||||
API_AUTHED_CACHE.insert(
|
||||
@@ -506,6 +513,8 @@ pub struct NewToken {
|
||||
pub impersonate_email: Option<String>,
|
||||
pub scopes: Option<Vec<String>>,
|
||||
pub workspace_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub read_only: Option<bool>,
|
||||
}
|
||||
|
||||
impl NewToken {
|
||||
@@ -515,8 +524,9 @@ impl NewToken {
|
||||
impersonate_email: Option<String>,
|
||||
scopes: Option<Vec<String>>,
|
||||
workspace_id: Option<String>,
|
||||
read_only: Option<bool>,
|
||||
) -> Self {
|
||||
Self { label, expiration, impersonate_email, scopes, workspace_id }
|
||||
Self { label, expiration, impersonate_email, scopes, workspace_id, read_only }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -564,8 +574,8 @@ pub async fn create_token_internal(
|
||||
}
|
||||
let rows = sqlx::query!(
|
||||
"INSERT INTO token
|
||||
(token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9
|
||||
(token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, workspace_id, read_only)
|
||||
SELECT $1, $2, $3, $4, $5, $6, $7, $8, $9, $10
|
||||
WHERE $9::varchar IS NULL OR NOT EXISTS(
|
||||
SELECT 1 FROM workspace WHERE id = $9 AND deleted = true
|
||||
)",
|
||||
@@ -578,6 +588,7 @@ pub async fn create_token_internal(
|
||||
is_super_admin,
|
||||
token_config.scopes.as_ref().map(|x| x.as_slice()),
|
||||
token_config.workspace_id,
|
||||
token_config.read_only.unwrap_or(false),
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
@@ -686,6 +686,19 @@ fn scope_grants_access(
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Enforces a token's `read_only` flag: only methods classified as `Read`
|
||||
/// (GET/HEAD/OPTIONS) are allowed. Run actions and mutating methods are
|
||||
/// rejected. Independent of `scopes`.
|
||||
pub fn check_read_only_for_route(route_path: &str, http_method: &str) -> Result<()> {
|
||||
if map_http_method_to_action(http_method, route_path) == ScopeAction::Read {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::PermissionDenied(
|
||||
"Token is read-only. Mutating endpoints are not allowed.".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to check if scopes allow access to a route
|
||||
pub fn check_scopes_for_route(
|
||||
token_scopes: Option<&[String]>,
|
||||
@@ -778,6 +791,34 @@ mod tests {
|
||||
assert_eq!(route_suffix, Some("flow_conversations/list".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_read_only_for_route() {
|
||||
// Plain GETs pass.
|
||||
assert!(check_read_only_for_route("/api/w/x/scripts/list", "GET").is_ok());
|
||||
assert!(check_read_only_for_route("/api/w/x/scripts/get/foo", "HEAD").is_ok());
|
||||
assert!(check_read_only_for_route("/api/w/x/anything", "OPTIONS").is_ok());
|
||||
|
||||
// Mutating methods are rejected.
|
||||
assert!(check_read_only_for_route("/api/w/x/scripts/create", "POST").is_err());
|
||||
assert!(check_read_only_for_route("/api/w/x/scripts/update", "PUT").is_err());
|
||||
assert!(check_read_only_for_route("/api/w/x/scripts/delete", "DELETE").is_err());
|
||||
assert!(check_read_only_for_route("/api/w/x/scripts/patch", "PATCH").is_err());
|
||||
|
||||
// Run paths are rejected even on GET (map_http_method_to_action elevates
|
||||
// them to Run via RUN_PATH_ACTIONS).
|
||||
assert!(check_read_only_for_route("/api/w/x/jobs/run/p/f/foo", "GET").is_err());
|
||||
assert!(check_read_only_for_route("/api/w/x/jobs/run/p/f/foo", "POST").is_err());
|
||||
|
||||
// OAuth/registration endpoints under /api/mcp/* must NOT be exempted by
|
||||
// the auth middleware — they go through this check on the gateway side
|
||||
// because they can mint non-read-only tokens. The middleware decides
|
||||
// which paths to exempt; this helper is method-only, so we just assert
|
||||
// that mutating methods still fail.
|
||||
assert!(
|
||||
check_read_only_for_route("/api/mcp/gateway/oauth/server/approve", "POST").is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_specific_scope_access() {
|
||||
let scopes = vec!["jobs:read".to_string()];
|
||||
|
||||
@@ -12,6 +12,9 @@ use axum::{
|
||||
Json, Router,
|
||||
};
|
||||
use windmill_api_auth::require_owner_of_path;
|
||||
use windmill_audit::audit_oss::audit_log;
|
||||
use windmill_audit::ActionKind;
|
||||
use windmill_common::scripts::ScriptHash;
|
||||
use windmill_common::DB;
|
||||
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
|
||||
|
||||
@@ -24,6 +27,31 @@ use windmill_common::{
|
||||
utils::{not_found_if_none, StripPath},
|
||||
};
|
||||
|
||||
/// Map a granular-ACL kind segment to the audit-log action prefix used by the
|
||||
/// per-kind CRUD endpoints (e.g. `flows.update`, `scripts.update`). The
|
||||
/// resulting action is suffixed with `.grant_acl` / `.revoke_acl` so the
|
||||
/// audit log keeps a per-resource record of every `/acls/*` mutation —
|
||||
/// folder/group already log via their dedicated permission-history tables.
|
||||
fn audit_action_prefix_for_acl_kind(kind: &str) -> Option<&'static str> {
|
||||
match kind {
|
||||
"script" => Some("scripts"),
|
||||
"flow" => Some("flows"),
|
||||
"app" => Some("apps"),
|
||||
// Distinct prefix so dashboards aggregating on `action` can separate
|
||||
// raw_app ACL mutations from regular app ones without parsing the
|
||||
// `kind` parameters field. (The granular_acls SQL routes raw_app
|
||||
// writes to the same `app` table; audit log identity is separate.)
|
||||
"raw_app" => Some("raw_apps"),
|
||||
"resource" => Some("resources"),
|
||||
"variable" => Some("variables"),
|
||||
"schedule" => Some("schedules"),
|
||||
"http_trigger" | "websocket_trigger" | "kafka_trigger" | "nats_trigger"
|
||||
| "postgres_trigger" | "mqtt_trigger" | "gcp_trigger" | "azure_trigger" | "sqs_trigger"
|
||||
| "email_trigger" => Some("triggers"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
const KINDS: [&str; 20] = [
|
||||
"script",
|
||||
"group_",
|
||||
@@ -131,9 +159,15 @@ async fn add_granular_acl(
|
||||
}
|
||||
}
|
||||
|
||||
// v2 raw apps are stored in the `app` table (with `app_version.raw_app = true`
|
||||
// distinguishing them from regular apps); the legacy `raw_app` table no longer
|
||||
// backs the workspace export, so granting/revoking on `raw_app` must hit `app`
|
||||
// for the change to be visible. Git-sync dispatch still uses
|
||||
// DeployedObject::RawApp so the worker writes back `<path>.raw_app.json`.
|
||||
let table = if kind == "raw_app" { "app" } else { kind };
|
||||
// SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function.
|
||||
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
|
||||
"UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2), \
|
||||
"UPDATE {table} SET extra_perms = jsonb_set(extra_perms, $1, to_jsonb($2), \
|
||||
true) WHERE {identifier} = $3 AND workspace_id = $4 RETURNING extra_perms"
|
||||
))
|
||||
.bind(vec![owner.clone()])
|
||||
@@ -175,6 +209,34 @@ async fn add_granular_acl(
|
||||
Some(&owner),
|
||||
)
|
||||
.await?;
|
||||
} else if let Some(prefix) = audit_action_prefix_for_acl_kind(kind) {
|
||||
// Mirror the folder/group permission-history coverage for every other
|
||||
// ACLable kind. Folder/group already wrote a dedicated history row
|
||||
// above; everything else (script/flow/app/raw_app/resource/...) lands
|
||||
// in the general audit_log table here.
|
||||
let access = if write.unwrap_or(false) {
|
||||
"write"
|
||||
} else {
|
||||
"read"
|
||||
};
|
||||
let action = format!("{}.grant_acl", prefix);
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed,
|
||||
action.as_str(),
|
||||
ActionKind::Update,
|
||||
&w_id,
|
||||
Some(path),
|
||||
Some(
|
||||
[
|
||||
("kind", kind),
|
||||
("owner", owner.as_str()),
|
||||
("access", access),
|
||||
]
|
||||
.into(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
tx.commit().await?;
|
||||
@@ -193,46 +255,67 @@ async fn add_granular_acl(
|
||||
)
|
||||
.await?
|
||||
}
|
||||
// "app" => {
|
||||
// handle_deployment_metadata(
|
||||
// &authed.email,
|
||||
// &authed.username,
|
||||
// &db,
|
||||
// &w_id,
|
||||
// DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
|
||||
// Some(format!("App '{}' changed permissions", path)),
|
||||
// // true,
|
||||
// )
|
||||
// .await?
|
||||
// }
|
||||
// "script" => {
|
||||
// handle_deployment_metadata(
|
||||
// &authed.email,
|
||||
// &authed.username,
|
||||
// &db,
|
||||
// &w_id,
|
||||
// DeployedObject::Script {
|
||||
// path: path.to_string(),
|
||||
// parent_path: None,
|
||||
// hash: ScriptHash(0),
|
||||
// },
|
||||
// Some(format!("Script '{}' changed permissions", path)),
|
||||
// // true,
|
||||
// )
|
||||
// .await?
|
||||
// }
|
||||
// "flow" => {
|
||||
// handle_deployment_metadata(
|
||||
// &authed.email,
|
||||
// &authed.username,
|
||||
// &db,
|
||||
// &w_id,
|
||||
// DeployedObject::Flow { path: path.to_string(), parent_path: None },
|
||||
// Some(format!("Flow '{}' changed permissions", path)),
|
||||
// // true,
|
||||
// )
|
||||
// .await?
|
||||
// }
|
||||
"app" => {
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
|
||||
Some(format!("App '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
"raw_app" => {
|
||||
// RawApp deliberately uses its own DeployedObject variant: the
|
||||
// git-sync worker reads `path_type` ("app" vs "raw_app") to decide
|
||||
// whether to write `<path>.app.json` or `<path>.raw_app.json`.
|
||||
// Collapsing this into `App` would dispatch raw_app perm changes
|
||||
// against the wrong file in the repo.
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::RawApp { path: path.to_string(), parent_path: None, version: 0 },
|
||||
Some(format!("Raw App '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
"script" => {
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::Script {
|
||||
path: path.to_string(),
|
||||
parent_path: None,
|
||||
hash: ScriptHash(0),
|
||||
},
|
||||
Some(format!("Script '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
"flow" => {
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::Flow { path: path.to_string(), parent_path: None, version: 0 },
|
||||
Some(format!("Flow '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
@@ -295,13 +378,17 @@ async fn remove_granular_acl(
|
||||
require_owner_of_path(&authed, path)?;
|
||||
}
|
||||
|
||||
// See add_granular_acl: kind="raw_app" must hit the `app` table because v2
|
||||
// raw apps live there and the legacy `raw_app` table no longer backs the
|
||||
// workspace export.
|
||||
let table = if kind == "raw_app" { "app" } else { kind };
|
||||
// SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function.
|
||||
let obj_o = sqlx::query_scalar::<_, bool>(&format!(
|
||||
"WITH old AS (
|
||||
SELECT extra_perms->$1 as old_write FROM {kind}
|
||||
SELECT extra_perms->$1 as old_write FROM {table}
|
||||
WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1
|
||||
)
|
||||
UPDATE {kind} SET extra_perms = extra_perms - $1
|
||||
UPDATE {table} SET extra_perms = extra_perms - $1
|
||||
WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1
|
||||
RETURNING (SELECT old_write FROM old)::bool"
|
||||
))
|
||||
@@ -335,6 +422,28 @@ async fn remove_granular_acl(
|
||||
Some(&owner),
|
||||
)
|
||||
.await?;
|
||||
} else if let Some(prefix) = audit_action_prefix_for_acl_kind(kind) {
|
||||
// Mirror the add path: standard audit_log row for every kind that
|
||||
// doesn't have a dedicated permission-history table.
|
||||
let access = if write { "write" } else { "read" };
|
||||
let action = format!("{}.revoke_acl", prefix);
|
||||
audit_log(
|
||||
&mut *tx,
|
||||
&authed,
|
||||
action.as_str(),
|
||||
ActionKind::Update,
|
||||
&w_id,
|
||||
Some(path),
|
||||
Some(
|
||||
[
|
||||
("kind", kind),
|
||||
("owner", owner.as_str()),
|
||||
("access", access),
|
||||
]
|
||||
.into(),
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
tx.commit().await?;
|
||||
@@ -353,46 +462,68 @@ async fn remove_granular_acl(
|
||||
)
|
||||
.await?
|
||||
}
|
||||
// "app" => {
|
||||
// handle_deployment_metadata(
|
||||
// &authed.email,
|
||||
// &authed.username,
|
||||
// &db,
|
||||
// &w_id,
|
||||
// DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
|
||||
// Some(format!("App '{}' changed permissions", path)),
|
||||
// // true,
|
||||
// )
|
||||
// .await?
|
||||
// }
|
||||
// "script" => {
|
||||
// handle_deployment_metadata(
|
||||
// &authed.email,
|
||||
// &authed.username,
|
||||
// &db,
|
||||
// &w_id,
|
||||
// DeployedObject::Script {
|
||||
// path: path.to_string(),
|
||||
// parent_path: None,
|
||||
// hash: ScriptHash(0),
|
||||
// },
|
||||
// Some(format!("Script '{}' changed permissions", path)),
|
||||
// // true,
|
||||
// )
|
||||
// .await?
|
||||
// }
|
||||
// "flow" => {
|
||||
// handle_deployment_metadata(
|
||||
// &authed.email,
|
||||
// &authed.username,
|
||||
// &db,
|
||||
// &w_id,
|
||||
// DeployedObject::Flow { path: path.to_string(), parent_path: None },
|
||||
// Some(format!("Flow '{}' changed permissions", path)),
|
||||
// // true,
|
||||
// )
|
||||
// .await?
|
||||
// }
|
||||
"app" => {
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::App { path: path.to_string(), parent_path: None, version: 0 },
|
||||
Some(format!("App '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
"raw_app" => {
|
||||
// See add_granular_acl: raw_app must use its own DeployedObject
|
||||
// variant so git-sync writes `<path>.raw_app.json`.
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::RawApp {
|
||||
path: path.to_string(),
|
||||
parent_path: None,
|
||||
version: 0,
|
||||
},
|
||||
Some(format!("Raw App '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
"script" => {
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::Script {
|
||||
path: path.to_string(),
|
||||
parent_path: None,
|
||||
hash: ScriptHash(0),
|
||||
},
|
||||
Some(format!("Script '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
"flow" => {
|
||||
handle_deployment_metadata(
|
||||
&authed.email,
|
||||
&authed.username,
|
||||
&db,
|
||||
&w_id,
|
||||
DeployedObject::Flow { path: path.to_string(), parent_path: None, version: 0 },
|
||||
Some(format!("Flow '{}' changed permissions", path)),
|
||||
true,
|
||||
None,
|
||||
)
|
||||
.await?
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
@@ -421,9 +552,13 @@ async fn get_granular_acls(
|
||||
} else {
|
||||
"path"
|
||||
};
|
||||
// See add_granular_acl: raw_app rows live in the `app` table now, so the
|
||||
// read path must also target `app` — otherwise GET would return stale or
|
||||
// 404 state while POST /acls/add and /acls/remove write to `app`.
|
||||
let table = if kind == "raw_app" { "app" } else { kind };
|
||||
// SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function.
|
||||
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
|
||||
"SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2"
|
||||
"SELECT extra_perms from {table} WHERE {identifier} = $1 AND workspace_id = $2"
|
||||
))
|
||||
.bind(path)
|
||||
.bind(w_id)
|
||||
|
||||
@@ -51,6 +51,7 @@ fn test_authed() -> ApiAuthed {
|
||||
scopes: None,
|
||||
username_override: None,
|
||||
token_prefix: None,
|
||||
read_only: false,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -573,8 +573,10 @@ async fn test_promotion_individual_branch_debounces_per_path(
|
||||
// Wait for both deployment callbacks to resolve a debounce key. The
|
||||
// alpha key is polled first as a warm-up, then we also wait for beta
|
||||
// so the assertions don't race the second spawned callback.
|
||||
let expected_alpha = "git_sync:script:f/target/alpha";
|
||||
let expected_beta = "git_sync:script:f/target/beta";
|
||||
// Keys are namespaced by the repo's resource path so multiple promotion
|
||||
// repos don't collide on the same key.
|
||||
let expected_alpha = "git_sync:$res:u/test-user/test_git_repo:script:f/target/alpha";
|
||||
let expected_beta = "git_sync:$res:u/test-user/test_git_repo:script:f/target/beta";
|
||||
let _ = wait_for_debounce_key(&db, expected_alpha, Duration::from_secs(5)).await?;
|
||||
let keys = wait_for_debounce_key(&db, expected_beta, Duration::from_secs(5)).await?;
|
||||
assert!(
|
||||
@@ -589,6 +591,162 @@ async fn test_promotion_individual_branch_debounces_per_path(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a second git repository resource for multi-repo tests.
|
||||
#[allow(dead_code)]
|
||||
async fn create_second_git_repo_resource(db: &Pool<Postgres>) -> anyhow::Result<()> {
|
||||
sqlx::query(
|
||||
r#"
|
||||
INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by)
|
||||
VALUES ('test-workspace', 'u/test-user/test_git_repo_2', $1::jsonb, 'git_repository', '{}'::jsonb, 'test-user')
|
||||
ON CONFLICT (workspace_id, path) DO NOTHING
|
||||
"#,
|
||||
)
|
||||
.bind(json!({
|
||||
"url": "https://github.com/test/test2.git",
|
||||
"branch": "main",
|
||||
"token": "test-token-2"
|
||||
}))
|
||||
.execute(db)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Configure git sync with TWO promotion-mode repositories pointing at distinct
|
||||
/// git repo resources. Both repos use the same sync script and the same item
|
||||
/// filters — they only differ in the repo they target.
|
||||
#[allow(dead_code)]
|
||||
async fn setup_two_promotion_repos_config(
|
||||
db: &Pool<Postgres>,
|
||||
sync_script_path: &str,
|
||||
group_by_folder: bool,
|
||||
) -> anyhow::Result<()> {
|
||||
let git_sync_config = json!({
|
||||
"include_type": ["script"],
|
||||
"include_path": ["**"],
|
||||
"repositories": [
|
||||
{
|
||||
"script_path": sync_script_path,
|
||||
"git_repo_resource_path": "$res:u/test-user/test_git_repo",
|
||||
"use_individual_branch": true,
|
||||
"group_by_folder": group_by_folder
|
||||
},
|
||||
{
|
||||
"script_path": sync_script_path,
|
||||
"git_repo_resource_path": "$res:u/test-user/test_git_repo_2",
|
||||
"use_individual_branch": true,
|
||||
"group_by_folder": group_by_folder
|
||||
}
|
||||
]
|
||||
});
|
||||
|
||||
sqlx::query!(
|
||||
"UPDATE workspace_settings SET git_sync = $1 WHERE workspace_id = $2",
|
||||
git_sync_config,
|
||||
"test-workspace"
|
||||
)
|
||||
.execute(db)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression test for: when two promotion-mode repos are configured with the
|
||||
/// same `use_individual_branch=true` settings (i.e. a primary and a secondary
|
||||
/// promotion repo), deploying a single script must enqueue ONE deployment
|
||||
/// callback per repo. Both callbacks must remain in the queue — neither may
|
||||
/// be debounced into oblivion by the other.
|
||||
///
|
||||
/// The bug this guards against: the debounce key for promotion mode was
|
||||
/// derived only from (path_type, path) and omitted any per-repo identifier,
|
||||
/// so the second repo's push hit ON CONFLICT in `upsert_debounce_key` and
|
||||
/// `complete_debounced_job` flagged the first repo's job as `status='skipped'`
|
||||
/// — silently dropping one of the two pushes.
|
||||
#[cfg(all(feature = "enterprise", feature = "private"))]
|
||||
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
|
||||
async fn test_two_promotion_repos_both_enqueue_callback(db: Pool<Postgres>) -> anyhow::Result<()> {
|
||||
initialize_tracing().await;
|
||||
|
||||
create_folder(&db, "28103").await?;
|
||||
create_folder(&db, "target").await?;
|
||||
create_git_repo_resource(&db).await?;
|
||||
create_second_git_repo_resource(&db).await?;
|
||||
let sync_script_path = "f/28103/test_sync_two_promotion_repos";
|
||||
create_sync_script(&db, sync_script_path).await?;
|
||||
setup_two_promotion_repos_config(&db, sync_script_path, false).await?;
|
||||
|
||||
let (client, _port, _server) = init_client(db.clone()).await;
|
||||
|
||||
// Deploy a single script — handle_deployment_metadata should iterate
|
||||
// both repos and create one callback job per repo.
|
||||
create_test_script(&client, "f/target/alpha").await?;
|
||||
|
||||
// Both callbacks should reach the queue. With the bug, only one survives
|
||||
// (the other is moved to v2_job_completed with status='skipped').
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(5);
|
||||
let mut last_jobs: Vec<DeploymentCallbackJob> = vec![];
|
||||
loop {
|
||||
last_jobs =
|
||||
get_deployment_callback_jobs(&db, sync_script_path, Duration::from_millis(200)).await?;
|
||||
if last_jobs.len() >= 2 {
|
||||
break;
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
|
||||
// Inspect what landed in v2_job_completed so failure messages explain why.
|
||||
let skipped: Vec<(uuid::Uuid, String)> = sqlx::query_as(
|
||||
r#"
|
||||
SELECT c.id, c.status::text
|
||||
FROM v2_job_completed c
|
||||
JOIN v2_job j ON j.id = c.id
|
||||
WHERE j.runnable_path = $1 AND j.kind = 'deploymentcallback'
|
||||
"#,
|
||||
)
|
||||
.bind(sync_script_path)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
last_jobs.len(),
|
||||
2,
|
||||
"expected 2 deployment callback jobs in v2_job_queue (one per promotion repo), got {} queued + {:?} completed",
|
||||
last_jobs.len(),
|
||||
skipped,
|
||||
);
|
||||
|
||||
// Per-repo args sanity check: the two jobs must target different repos.
|
||||
let mut repo_paths: Vec<String> = last_jobs
|
||||
.iter()
|
||||
.filter_map(|j| {
|
||||
j.args
|
||||
.as_ref()
|
||||
.and_then(|a| a.get("repo_url_resource_path"))
|
||||
.and_then(|v| v.as_str())
|
||||
.map(|s| s.to_string())
|
||||
})
|
||||
.collect();
|
||||
repo_paths.sort();
|
||||
repo_paths.dedup();
|
||||
assert_eq!(
|
||||
repo_paths.len(),
|
||||
2,
|
||||
"expected callbacks to target two distinct repos, got: {:?}",
|
||||
last_jobs.iter().map(|j| &j.args).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
// No callback should have been silently skipped via debouncing collision.
|
||||
assert!(
|
||||
skipped.iter().all(|(_, s)| s != "skipped"),
|
||||
"no deployment callback should be marked skipped, got: {:?}",
|
||||
skipped,
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Promotion mode with group_by_folder: items destined for the same per-folder
|
||||
/// branch must share one debounce key so they accumulate into a single sync
|
||||
/// job; scripts in different folders must get distinct keys.
|
||||
@@ -615,8 +773,9 @@ async fn test_promotion_group_by_folder_debounces_per_folder(
|
||||
// One in a different folder — should get its own key.
|
||||
create_test_script(&client, "f/other/gamma").await?;
|
||||
|
||||
let expected_grouped = "git_sync:folder:f/grouped";
|
||||
let expected_other = "git_sync:folder:f/other";
|
||||
// Keys are namespaced by the repo's resource path.
|
||||
let expected_grouped = "git_sync:$res:u/test-user/test_git_repo:folder:f/grouped";
|
||||
let expected_other = "git_sync:$res:u/test-user/test_git_repo:folder:f/other";
|
||||
// Wait for BOTH folder keys to appear, not just the first one.
|
||||
let keys = wait_for_debounce_key(&db, expected_other, Duration::from_secs(5)).await?;
|
||||
assert!(
|
||||
@@ -629,7 +788,9 @@ async fn test_promotion_group_by_folder_debounces_per_folder(
|
||||
);
|
||||
// Paths within the same folder must NOT leak as their own keys.
|
||||
assert!(
|
||||
!keys.iter().any(|k| k.starts_with("git_sync:script:")),
|
||||
!keys
|
||||
.iter()
|
||||
.any(|k| k.contains(":script:f/grouped/") || k.contains(":script:f/other/")),
|
||||
"group_by_folder mode should not emit per-path keys, got: {keys:?}"
|
||||
);
|
||||
|
||||
|
||||
@@ -980,12 +980,10 @@ async fn create_script_internal<'c>(
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
}
|
||||
let clashing_script = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(
|
||||
&format!(
|
||||
"SELECT {} FROM script WHERE path = $1 AND archived = false AND workspace_id = $2",
|
||||
windmill_common::scripts::SCRIPT_COLUMNS,
|
||||
),
|
||||
)
|
||||
let clashing_script = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(&format!(
|
||||
"SELECT {} FROM script WHERE path = $1 AND archived = false AND workspace_id = $2",
|
||||
windmill_common::scripts::SCRIPT_COLUMNS,
|
||||
))
|
||||
.bind(&ns.path)
|
||||
.bind(&w_id)
|
||||
.fetch_optional(&mut *tx)
|
||||
@@ -1218,6 +1216,22 @@ async fn create_script_internal<'c>(
|
||||
}
|
||||
};
|
||||
|
||||
// Failure, Trigger, and Approval scripts are runnable entrypoints by
|
||||
// definition. They must never be marked `auto_kind = 'lib'`, or they
|
||||
// disappear from the flow error-handler / trigger / approval pickers
|
||||
// (which filter out lib scripts). Strip a stray `lib` here so a parser
|
||||
// misclassification — e.g. failing to detect `main` after a deno_ast
|
||||
// bump — cannot orphan these scripts in the UI.
|
||||
let auto_kind = if matches!(
|
||||
ns.kind,
|
||||
Some(ScriptKind::Failure) | Some(ScriptKind::Trigger) | Some(ScriptKind::Approval)
|
||||
) && auto_kind.as_deref() == Some("lib")
|
||||
{
|
||||
None
|
||||
} else {
|
||||
auto_kind
|
||||
};
|
||||
|
||||
let ci_test_refs =
|
||||
windmill_common::schema::parse_ci_test_annotation(&ns.content, &lang.as_comment_lit());
|
||||
// `pipeline` wins over `test` and any client-supplied auto_kind. The
|
||||
@@ -2298,7 +2312,7 @@ async fn exists_script_by_path(
|
||||
let path = path.to_path();
|
||||
|
||||
let exists = sqlx::query_scalar!(
|
||||
"SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1)",
|
||||
"SELECT EXISTS(SELECT 1 FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1)",
|
||||
path,
|
||||
w_id
|
||||
)
|
||||
@@ -2332,12 +2346,10 @@ async fn get_script_by_hash_internal<'c>(
|
||||
.fetch_optional(&mut **db)
|
||||
.await?
|
||||
} else {
|
||||
sqlx::query_as::<_, ScriptWithStarred<ScriptRunnableSettingsHandle>>(
|
||||
&format!(
|
||||
"SELECT {}, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
windmill_common::scripts::SCRIPT_COLUMNS,
|
||||
),
|
||||
)
|
||||
sqlx::query_as::<_, ScriptWithStarred<ScriptRunnableSettingsHandle>>(&format!(
|
||||
"SELECT {}, NULL as starred FROM script WHERE hash = $1 AND workspace_id = $2",
|
||||
windmill_common::scripts::SCRIPT_COLUMNS,
|
||||
))
|
||||
.bind(hash)
|
||||
.bind(workspace_id)
|
||||
.fetch_optional(&mut **db)
|
||||
|
||||
@@ -210,6 +210,8 @@ pub struct GlobalUserInfo {
|
||||
first_time_user: bool,
|
||||
role_source: String,
|
||||
disabled: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
workspace_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
@@ -297,6 +299,7 @@ pub struct TruncatedToken {
|
||||
pub last_used_at: chrono::DateTime<chrono::Utc>,
|
||||
pub scopes: Option<Vec<String>>,
|
||||
pub workspace_id: Option<String>,
|
||||
pub read_only: bool,
|
||||
}
|
||||
|
||||
// NewToken is re-exported from windmill-api-auth above
|
||||
@@ -450,13 +453,17 @@ async fn list_users_as_super_admin(
|
||||
let rows = if active_only.is_some_and(|x| x) {
|
||||
sqlx::query_as!(
|
||||
GlobalUserInfo,
|
||||
"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
|
||||
r#"WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),
|
||||
authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)
|
||||
SELECT email, email NOT IN (SELECT email FROM authors) as operator_only, login_type::text, verified, super_admin, devops, name, company, username, first_time_user, role_source, disabled
|
||||
SELECT email as "email!", (email NOT IN (SELECT email FROM authors)) as operator_only, login_type::text, verified as "verified!", super_admin as "super_admin!", devops as "devops!", name, company, username, first_time_user as "first_time_user!", role_source as "role_source!", disabled as "disabled!", NULL::text as workspace_id
|
||||
FROM password
|
||||
WHERE email IN (SELECT email FROM active_users)
|
||||
ORDER BY super_admin DESC, devops DESC
|
||||
LIMIT $1 OFFSET $2",
|
||||
UNION ALL
|
||||
SELECT email as "email!", true as operator_only, 'service_account'::text as login_type, true as "verified!", false as "super_admin!", false as "devops!", NULL::text as name, NULL::text as company, username, false as "first_time_user!", 'service_account'::text as "role_source!", disabled as "disabled!", workspace_id
|
||||
FROM usr
|
||||
WHERE is_service_account IS true
|
||||
ORDER BY "super_admin!" DESC, "devops!" DESC
|
||||
LIMIT $1 OFFSET $2"#,
|
||||
per_page as i32,
|
||||
offset as i32
|
||||
)
|
||||
@@ -465,8 +472,13 @@ async fn list_users_as_super_admin(
|
||||
} else {
|
||||
sqlx::query_as!(
|
||||
GlobalUserInfo,
|
||||
"SELECT email, login_type::text, verified, super_admin, devops, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password ORDER BY super_admin DESC, devops DESC, email LIMIT \
|
||||
$1 OFFSET $2",
|
||||
r#"SELECT email as "email!", login_type::text, verified as "verified!", super_admin as "super_admin!", devops as "devops!", name, company, username, NULL::bool as operator_only, first_time_user as "first_time_user!", role_source as "role_source!", disabled as "disabled!", NULL::text as workspace_id FROM password
|
||||
UNION ALL
|
||||
SELECT email as "email!", 'service_account'::text as login_type, true as "verified!", false as "super_admin!", false as "devops!", NULL::text as name, NULL::text as company, username, true as operator_only, false as "first_time_user!", 'service_account'::text as "role_source!", disabled as "disabled!", workspace_id
|
||||
FROM usr
|
||||
WHERE is_service_account IS true
|
||||
ORDER BY "super_admin!" DESC, "devops!" DESC, "email!"
|
||||
LIMIT $1 OFFSET $2"#,
|
||||
per_page as i32,
|
||||
offset as i32
|
||||
)
|
||||
@@ -715,7 +727,7 @@ async fn global_whoami(
|
||||
) -> JsonResult<GlobalUserInfo> {
|
||||
let user = sqlx::query_as!(
|
||||
GlobalUserInfo,
|
||||
"SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled FROM password WHERE \
|
||||
"SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, first_time_user, role_source, disabled, NULL::text as workspace_id FROM password WHERE \
|
||||
email = $1",
|
||||
email
|
||||
)
|
||||
@@ -739,6 +751,7 @@ async fn global_whoami(
|
||||
first_time_user: false,
|
||||
role_source: "manual".to_string(),
|
||||
disabled: false,
|
||||
workspace_id: None,
|
||||
}))
|
||||
} else {
|
||||
// Service accounts don't have a password row
|
||||
@@ -755,6 +768,7 @@ async fn global_whoami(
|
||||
first_time_user: false,
|
||||
role_source: "service_account".to_string(),
|
||||
disabled: false,
|
||||
workspace_id: None,
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -2249,7 +2263,7 @@ async fn list_tokens(
|
||||
sqlx::query_as!(
|
||||
TruncatedToken,
|
||||
"SELECT label, token_prefix, expiration, created_at, \
|
||||
last_used_at, scopes, workspace_id FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)
|
||||
last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1 AND (label != 'ephemeral-script' OR label IS NULL)
|
||||
ORDER BY created_at DESC LIMIT $2 OFFSET $3",
|
||||
email,
|
||||
per_page as i64,
|
||||
@@ -2261,7 +2275,7 @@ async fn list_tokens(
|
||||
sqlx::query_as!(
|
||||
TruncatedToken,
|
||||
"SELECT label, token_prefix, expiration, created_at, \
|
||||
last_used_at, scopes, workspace_id FROM token WHERE email = $1
|
||||
last_used_at, scopes, workspace_id, read_only FROM token WHERE email = $1
|
||||
ORDER BY created_at DESC LIMIT $2 OFFSET $3",
|
||||
email,
|
||||
per_page as i64,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
openapi: "3.0.3"
|
||||
|
||||
info:
|
||||
version: 1.699.0
|
||||
version: 1.702.1
|
||||
title: Windmill API
|
||||
|
||||
contact:
|
||||
@@ -22631,10 +22631,13 @@ components:
|
||||
type: string
|
||||
workspace_id:
|
||||
type: string
|
||||
read_only:
|
||||
type: boolean
|
||||
required:
|
||||
- token_prefix
|
||||
- created_at
|
||||
- last_used_at
|
||||
- read_only
|
||||
|
||||
ExternalJwtToken:
|
||||
type: object
|
||||
@@ -22683,6 +22686,12 @@ components:
|
||||
type: string
|
||||
workspace_id:
|
||||
type: string
|
||||
read_only:
|
||||
type: boolean
|
||||
description: |
|
||||
If true, the token is restricted to read-only HTTP methods
|
||||
(GET/HEAD/OPTIONS). Mutating endpoints and job-run actions are
|
||||
rejected with 403, regardless of the scopes attached.
|
||||
|
||||
NewTokenImpersonate:
|
||||
type: object
|
||||
@@ -26355,7 +26364,7 @@ components:
|
||||
type: string
|
||||
login_type:
|
||||
type: string
|
||||
enum: ["password", "github"]
|
||||
enum: ["password", "github", "service_account"]
|
||||
super_admin:
|
||||
type: boolean
|
||||
devops:
|
||||
@@ -26374,9 +26383,11 @@ components:
|
||||
type: boolean
|
||||
role_source:
|
||||
type: string
|
||||
enum: ["manual", "instance_group"]
|
||||
enum: ["manual", "instance_group", "service_account"]
|
||||
disabled:
|
||||
type: boolean
|
||||
workspace_id:
|
||||
type: string
|
||||
|
||||
required:
|
||||
- email
|
||||
|
||||
@@ -330,6 +330,7 @@ async fn inject_agent_authed(
|
||||
scopes: None,
|
||||
username_override: None,
|
||||
token_prefix: None,
|
||||
read_only: false,
|
||||
},
|
||||
job_id: None,
|
||||
});
|
||||
|
||||
@@ -7,6 +7,7 @@ use async_trait::async_trait;
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
use windmill_common::{db::UserDB, utils::StripPath, DB};
|
||||
use windmill_mcp::common::schema::enrich_resource_schemas;
|
||||
use windmill_mcp::common::transform::apply_key_transformation;
|
||||
use windmill_mcp::common::types::{
|
||||
FlowInfo, HubScriptInfo, ResourceInfo, ResourceType, SchemaType, ScriptInfo,
|
||||
@@ -203,71 +204,13 @@ impl McpBackend for WindmillBackend {
|
||||
schema_obj.properties.insert(new_key, value);
|
||||
}
|
||||
|
||||
for (_key, prop_value) in schema_obj.properties.iter_mut() {
|
||||
if let Value::Object(prop_map) = prop_value {
|
||||
if let Some(format_value) = prop_map.get("format") {
|
||||
if let Value::String(format_str) = format_value {
|
||||
if format_str.starts_with("resource-") {
|
||||
let resource_type_key =
|
||||
format_str.split("-").last().unwrap_or_default().to_string();
|
||||
let resource_type = resources_types
|
||||
.iter()
|
||||
.find(|rt| rt.name == resource_type_key);
|
||||
let resource_type_obj = resource_type.cloned();
|
||||
|
||||
if let Some(resource_cache) = resources_cache.get(&resource_type_key) {
|
||||
let resources_count = resource_cache.len();
|
||||
let description = match resource_type_obj {
|
||||
Some(resource_type_obj) => format!(
|
||||
"This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}",
|
||||
resource_type_obj.name,
|
||||
resource_type_obj.description.as_deref().unwrap_or("No description"),
|
||||
if resources_count == 0 {
|
||||
"This resource does not have any available instances, you should create one from your windmill workspace."
|
||||
} else if resources_count > 1 {
|
||||
"This resource has multiple available instances, you should precisely select the one you want to use."
|
||||
} else {
|
||||
"There is 1 resource available."
|
||||
}
|
||||
),
|
||||
None => "An object parameter.".to_string(),
|
||||
};
|
||||
prop_map.insert(
|
||||
"type".to_string(),
|
||||
Value::String("string".to_string()),
|
||||
);
|
||||
prop_map
|
||||
.insert("description".to_string(), Value::String(description));
|
||||
if resources_count > 0 {
|
||||
let resources_description = resource_cache
|
||||
.iter()
|
||||
.map(|resource| {
|
||||
format!(
|
||||
"{}: $res:{}",
|
||||
resource
|
||||
.description
|
||||
.as_deref()
|
||||
.unwrap_or("No title"),
|
||||
resource.path
|
||||
)
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
.join("\\n");
|
||||
|
||||
prop_map.insert(
|
||||
"description".to_string(),
|
||||
Value::String(format!(
|
||||
"{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}",
|
||||
prop_map.get("description").unwrap_or(&Value::String("No description".to_string())),
|
||||
resources_description
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Enrich every resource reference in the schema — including those
|
||||
// inside `items`, nested `properties`, etc. — with a description
|
||||
// listing the available resources. Both shapes are handled:
|
||||
// { type: "object", format: "resource-<name>" } (top-level scalar)
|
||||
// { type: "resource", resourceType: "<name>" } (inside list items)
|
||||
for prop_value in schema_obj.properties.values_mut() {
|
||||
enrich_resource_schemas(prop_value, resources_cache, resources_types);
|
||||
}
|
||||
|
||||
schema_obj
|
||||
|
||||
@@ -110,6 +110,12 @@ struct ScriptMetadata {
|
||||
pub debouncing_settings: DebouncingSettings,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub labels: Option<Vec<String>>,
|
||||
#[serde(skip_serializing_if = "is_empty_extra_perms")]
|
||||
pub extra_perms: serde_json::Value,
|
||||
}
|
||||
|
||||
fn is_empty_extra_perms(value: &serde_json::Value) -> bool {
|
||||
value.as_object().is_some_and(|o| o.is_empty()) || value.is_null()
|
||||
}
|
||||
|
||||
pub fn is_none_or_false(val: &Option<bool>) -> bool {
|
||||
@@ -224,12 +230,34 @@ pub(crate) struct ArchiveQueryParams {
|
||||
default_ts: Option<String>,
|
||||
/// Settings format version: "v1" (default) returns legacy flat format, "v2" returns grouped format
|
||||
settings_version: Option<String>,
|
||||
/// Opt-in: include `extra_perms` on flow / script / app rows. Default `false`
|
||||
/// so cross-workspace tarball imports do not carry over ACLs referring to
|
||||
/// identities that may not exist in the target workspace. `wmill sync pull`
|
||||
/// passes `true` to surface ACLs in the git-tracked yaml.
|
||||
preserve_extra_perms: Option<bool>,
|
||||
}
|
||||
|
||||
/// How to handle `extra_perms` in the serialized output.
|
||||
///
|
||||
/// * `Drop` — strip the field unconditionally (legacy behavior for
|
||||
/// types that have never carried ACLs in source).
|
||||
/// * `KeepEvenEmpty` — always keep the field, even when `{}`. Matches the
|
||||
/// pre-existing serialization for folders and groups so
|
||||
/// no customer sees a one-time noisy diff on upgrade.
|
||||
/// * `KeepIfNonEmpty` — keep when there is at least one entry, drop when `{}`
|
||||
/// or null. New surface for flow / script / app, which
|
||||
/// never carried ACLs in source before this change.
|
||||
#[derive(Clone, Copy)]
|
||||
pub enum ExtraPermsBehavior {
|
||||
Drop,
|
||||
KeepEvenEmpty,
|
||||
KeepIfNonEmpty,
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn to_string_without_metadata<T>(
|
||||
value: &T,
|
||||
preserve_extra_perms: bool,
|
||||
extra_perms: ExtraPermsBehavior,
|
||||
ignore_keys: Option<Vec<&str>>,
|
||||
) -> Result<String>
|
||||
where
|
||||
@@ -274,8 +302,19 @@ where
|
||||
o2.remove("on_behalf_of");
|
||||
o2.remove("on_behalf_of_email");
|
||||
}
|
||||
if !preserve_extra_perms && obj.contains_key("extra_perms") {
|
||||
obj.remove("extra_perms");
|
||||
if obj.contains_key("extra_perms") {
|
||||
let is_empty_extra_perms = obj
|
||||
.get("extra_perms")
|
||||
.map(|v| v.as_object().is_some_and(|o| o.is_empty()) || v.is_null())
|
||||
.unwrap_or(true);
|
||||
let drop = match extra_perms {
|
||||
ExtraPermsBehavior::Drop => true,
|
||||
ExtraPermsBehavior::KeepEvenEmpty => false,
|
||||
ExtraPermsBehavior::KeepIfNonEmpty => is_empty_extra_perms,
|
||||
};
|
||||
if drop {
|
||||
obj.remove("extra_perms");
|
||||
}
|
||||
}
|
||||
if obj
|
||||
.get("default_permissioned_as")
|
||||
@@ -442,6 +481,7 @@ pub(crate) async fn tarball_workspace(
|
||||
include_workspace_dependencies,
|
||||
default_ts,
|
||||
settings_version,
|
||||
preserve_extra_perms,
|
||||
}): Query<ArchiveQueryParams>,
|
||||
) -> Result<([(HeaderName, String); 2], impl IntoResponse)> {
|
||||
tracing::info!(
|
||||
@@ -452,6 +492,16 @@ pub(crate) async fn tarball_workspace(
|
||||
skip_resources
|
||||
);
|
||||
|
||||
// Opt-in behavior for surfacing per-resource ACLs on flow/app rows.
|
||||
// Folder and group rows have always carried `extra_perms` in source and
|
||||
// continue to do so unconditionally (`KeepEvenEmpty`) so existing
|
||||
// customer git repos see no one-time noisy diff.
|
||||
let new_kinds_extra_perms = if preserve_extra_perms.unwrap_or(false) {
|
||||
ExtraPermsBehavior::KeepIfNonEmpty
|
||||
} else {
|
||||
ExtraPermsBehavior::Drop
|
||||
};
|
||||
|
||||
let mut tx = user_db.begin(&authed).await?;
|
||||
|
||||
// Source-of-truth check for fork-ness: the workspace's parent_workspace_id
|
||||
@@ -498,7 +548,8 @@ pub(crate) async fn tarball_workspace(
|
||||
for folder in folders {
|
||||
archive
|
||||
.write_to_archive(
|
||||
&to_string_without_metadata(&folder, true, None).unwrap(),
|
||||
&to_string_without_metadata(&folder, ExtraPermsBehavior::KeepEvenEmpty, None)
|
||||
.unwrap(),
|
||||
&format!("f/{}/folder.meta.json", folder.name),
|
||||
)
|
||||
.await?;
|
||||
@@ -506,15 +557,13 @@ pub(crate) async fn tarball_workspace(
|
||||
}
|
||||
|
||||
{
|
||||
let scripts = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(
|
||||
&format!(
|
||||
"SELECT {} FROM script as o WHERE workspace_id = $1 AND archived = false
|
||||
let scripts = sqlx::query_as::<_, Script<ScriptRunnableSettingsHandle>>(&format!(
|
||||
"SELECT {} FROM script as o WHERE workspace_id = $1 AND archived = false
|
||||
AND (draft_only IS NULL OR draft_only = false)
|
||||
AND created_at = (select max(created_at) from script where path = o.path AND \
|
||||
workspace_id = $1)",
|
||||
windmill_common::scripts::SCRIPT_COLUMNS,
|
||||
),
|
||||
)
|
||||
windmill_common::scripts::SCRIPT_COLUMNS,
|
||||
))
|
||||
.bind(&w_id)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
@@ -587,6 +636,15 @@ pub(crate) async fn tarball_workspace(
|
||||
on_behalf_of_email: script.on_behalf_of_email,
|
||||
modules: script.modules,
|
||||
labels: script.labels,
|
||||
// Same opt-in contract as flow/app: the tarball only surfaces
|
||||
// ACLs when `?preserve_extra_perms=true`. Passing `Null` lets the
|
||||
// `is_empty_extra_perms` skip-serializer drop the field entirely.
|
||||
extra_perms: if matches!(new_kinds_extra_perms, ExtraPermsBehavior::KeepIfNonEmpty)
|
||||
{
|
||||
script.extra_perms
|
||||
} else {
|
||||
serde_json::Value::Null
|
||||
},
|
||||
};
|
||||
let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
|
||||
archive
|
||||
@@ -605,7 +663,8 @@ pub(crate) async fn tarball_workspace(
|
||||
.await?;
|
||||
|
||||
for resource in resources {
|
||||
let resource_str = &to_string_without_metadata(&resource, false, None).unwrap();
|
||||
let resource_str =
|
||||
&to_string_without_metadata(&resource, ExtraPermsBehavior::Drop, None).unwrap();
|
||||
archive
|
||||
.write_to_archive(&resource_str, &format!("{}.resource.json", resource.path))
|
||||
.await?;
|
||||
@@ -622,7 +681,9 @@ pub(crate) async fn tarball_workspace(
|
||||
.await?;
|
||||
|
||||
for resource_type in resource_types {
|
||||
let resource_str = &to_string_without_metadata(&resource_type, false, None).unwrap();
|
||||
let resource_str =
|
||||
&to_string_without_metadata(&resource_type, ExtraPermsBehavior::Drop, None)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&resource_str,
|
||||
@@ -644,7 +705,7 @@ pub(crate) async fn tarball_workspace(
|
||||
.await?;
|
||||
|
||||
for flow in flows {
|
||||
let flow_str = &to_string_without_metadata(&flow, false, None).unwrap();
|
||||
let flow_str = &to_string_without_metadata(&flow, new_kinds_extra_perms, None).unwrap();
|
||||
archive
|
||||
.write_to_archive(&flow_str, &format!("{}.flow.json", flow.path))
|
||||
.await?;
|
||||
@@ -673,7 +734,8 @@ pub(crate) async fn tarball_workspace(
|
||||
Error::internal_err(format!("Error decrypting variable {}: {}", var.path, e))
|
||||
})?);
|
||||
}
|
||||
let var_str = &to_string_without_metadata(&var, false, None).unwrap();
|
||||
let var_str =
|
||||
&to_string_without_metadata(&var, ExtraPermsBehavior::Drop, None).unwrap();
|
||||
archive
|
||||
.write_to_archive(&var_str, &format!("{}.variable.json", var.path))
|
||||
.await?;
|
||||
@@ -693,7 +755,7 @@ pub(crate) async fn tarball_workspace(
|
||||
.await?;
|
||||
|
||||
for app in apps {
|
||||
let app_str = &to_string_without_metadata(&app, false, None).unwrap();
|
||||
let app_str = &to_string_without_metadata(&app, new_kinds_extra_perms, None).unwrap();
|
||||
let kind = if app.raw_app { "raw_app" } else { "app" };
|
||||
archive
|
||||
.write_to_archive(&app_str, &format!("{}.{}.json", app.path, kind))
|
||||
@@ -711,7 +773,7 @@ pub(crate) async fn tarball_workspace(
|
||||
workspace_dependencies.len()
|
||||
);
|
||||
for dep in workspace_dependencies {
|
||||
// let dep_str = &to_string_without_metadata(&dep, false, None).unwrap();
|
||||
// let dep_str = &to_string_without_metadata(&dep, ExtraPermsBehavior::Drop, None).unwrap();
|
||||
let filename = WorkspaceDependencies::to_path(&dep.name, dep.language)?;
|
||||
tracing::info!(
|
||||
"Adding workspace dependency: name={:?}, language={:?}, filename={}",
|
||||
@@ -739,9 +801,12 @@ pub(crate) async fn tarball_workspace(
|
||||
|
||||
let schedule_ignore_keys = fork_schedule_ignore_keys(is_fork);
|
||||
for schedule in schedules {
|
||||
let app_str =
|
||||
&to_string_without_metadata(&schedule, false, schedule_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let app_str = &to_string_without_metadata(
|
||||
&schedule,
|
||||
ExtraPermsBehavior::Drop,
|
||||
schedule_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(&app_str, &format!("{}.schedule.json", schedule.path))
|
||||
.await?;
|
||||
@@ -777,9 +842,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let http_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in http_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -796,9 +864,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let websocket_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in websocket_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -815,9 +886,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let kafka_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in kafka_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -834,9 +908,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let sqs_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in sqs_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -853,9 +930,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let gcp_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in gcp_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -872,9 +952,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let azure_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in azure_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -891,9 +974,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let nats_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in nats_triggers {
|
||||
let trigger_str: &String =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str: &String = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -910,9 +996,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let postgres_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in postgres_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -929,9 +1018,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let mqtt_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in mqtt_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -948,9 +1040,12 @@ pub(crate) async fn tarball_workspace(
|
||||
let email_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?;
|
||||
|
||||
for trigger in email_triggers {
|
||||
let trigger_str =
|
||||
&to_string_without_metadata(&trigger, false, trigger_ignore_keys.clone())
|
||||
.unwrap();
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
ExtraPermsBehavior::Drop,
|
||||
trigger_ignore_keys.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(
|
||||
&trigger_str,
|
||||
@@ -978,7 +1073,7 @@ pub(crate) async fn tarball_workspace(
|
||||
for trigger in native_triggers {
|
||||
let trigger_str = &to_string_without_metadata(
|
||||
&trigger,
|
||||
false,
|
||||
ExtraPermsBehavior::Drop,
|
||||
Some(native_ignore_keys.clone()),
|
||||
)
|
||||
.unwrap();
|
||||
@@ -1021,7 +1116,9 @@ pub(crate) async fn tarball_workspace(
|
||||
disabled: user.disabled,
|
||||
email: user.email,
|
||||
};
|
||||
let user_str = &to_string_without_metadata(&user, false, Some(vec!["email"])).unwrap();
|
||||
let user_str =
|
||||
&to_string_without_metadata(&user, ExtraPermsBehavior::Drop, Some(vec!["email"]))
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(&user_str, &format!("users/{}.user.json", user.email))
|
||||
.await?;
|
||||
@@ -1081,7 +1178,9 @@ pub(crate) async fn tarball_workspace(
|
||||
admins,
|
||||
};
|
||||
|
||||
let group_str = &to_string_without_metadata(&group, true, None).unwrap();
|
||||
let group_str =
|
||||
&to_string_without_metadata(&group, ExtraPermsBehavior::KeepEvenEmpty, None)
|
||||
.unwrap();
|
||||
archive
|
||||
.write_to_archive(&group_str, &format!("groups/{}.group.json", group.name))
|
||||
.await?;
|
||||
|
||||
@@ -200,7 +200,15 @@ fn opaque_json_schema(_: &mut schemars::gen::SchemaGenerator) -> schemars::schem
|
||||
|
||||
/// Typed global settings with schema validation.
|
||||
/// Known settings have explicit fields; unknown settings pass through via `extra`.
|
||||
///
|
||||
/// `#[serde(remote = "Self")]` turns the derived (de)serializers into inherent
|
||||
/// associated functions so we can wrap them with the manual `Deserialize` impl
|
||||
/// below. The wrapper preserves explicit `null` values (which the typed
|
||||
/// `Option<T>` fields would otherwise silently drop on round-trip through
|
||||
/// `to_settings_map`) by stashing them in `extra`, where they survive
|
||||
/// re-serialization and reach `diff_global_settings` as proper deletes.
|
||||
#[derive(Deserialize, Serialize, Clone, Debug, Default)]
|
||||
#[serde(remote = "Self")]
|
||||
#[cfg_attr(feature = "instance_config_schema", derive(schemars::JsonSchema))]
|
||||
pub struct GlobalSettings {
|
||||
// Numeric settings
|
||||
@@ -373,6 +381,42 @@ pub struct GlobalSettings {
|
||||
pub extra: BTreeMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Serialize for GlobalSettings {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
Self::serialize(self, serializer)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for GlobalSettings {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
// Capture top-level keys explicitly set to `null` so they survive the
|
||||
// `to_settings_map` round-trip — typed `Option<T>` fields all use
|
||||
// `skip_serializing_if = "Option::is_none"`, which would otherwise
|
||||
// silently drop the null. We stash the null entries in `extra`, which
|
||||
// serializes them back out as `null` for `diff_global_settings` to
|
||||
// route to deletes.
|
||||
let mut value = serde_json::Value::deserialize(deserializer)?;
|
||||
let null_keys: Vec<String> = value
|
||||
.as_object()
|
||||
.map(|m| {
|
||||
m.iter()
|
||||
.filter_map(|(k, v)| v.is_null().then(|| k.clone()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if let Some(obj) = value.as_object_mut() {
|
||||
for k in &null_keys {
|
||||
obj.remove(k);
|
||||
}
|
||||
}
|
||||
let mut s = Self::deserialize(value).map_err(serde::de::Error::custom)?;
|
||||
for k in null_keys {
|
||||
s.extra.insert(k, serde_json::Value::Null);
|
||||
}
|
||||
Ok(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl GlobalSettings {
|
||||
/// Convert to a flat `BTreeMap` suitable for DB sync.
|
||||
pub fn to_settings_map(&self) -> BTreeMap<String, serde_json::Value> {
|
||||
@@ -572,6 +616,12 @@ pub struct OtelTracingProxySettings {
|
||||
pub enabled: bool,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub enabled_languages: Vec<ScriptLang>,
|
||||
/// Comma-separated list of host patterns injected as NO_PROXY into jobs so their HTTP
|
||||
/// clients bypass the local MITM tracing proxy. Independent of the worker's own
|
||||
/// NO_PROXY env (which governs the proxy's upstream relay). Use this for clients that
|
||||
/// pin their own CA (kubectl, helm, terraform providers, aws cli for EKS, etc.).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub no_proxy_hosts: Option<String>,
|
||||
}
|
||||
|
||||
/// Script language identifier (for instance config use).
|
||||
@@ -1930,6 +1980,101 @@ mod tests {
|
||||
// to_settings_map edge cases
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
/// Regression test for bulk-endpoint deletion of typed settings.
|
||||
///
|
||||
/// A naive `#[derive(Deserialize)]` would route
|
||||
/// `{"object_store_cache_config": null}` to the typed `Option<...>` field
|
||||
/// as `None`, and `skip_serializing_if = "Option::is_none"` would then
|
||||
/// strip it from `to_settings_map`, so `diff_global_settings` (Merge mode)
|
||||
/// would never see the deletion. The manual `Deserialize` impl on
|
||||
/// `GlobalSettings` captures explicit top-level nulls into `extra` so they
|
||||
/// survive the round-trip and reach `diff_global_settings` as proper
|
||||
/// deletes.
|
||||
#[test]
|
||||
fn explicit_null_on_typed_field_survives_round_trip() {
|
||||
let json = serde_json::json!({
|
||||
"object_store_cache_config": null,
|
||||
"secret_backend": null,
|
||||
"smtp_settings": null,
|
||||
"base_url": "https://x",
|
||||
});
|
||||
let settings: GlobalSettings =
|
||||
serde_json::from_value(json).expect("null should deserialize");
|
||||
assert!(settings.object_store_cache_config.is_none());
|
||||
assert!(settings.secret_backend.is_none());
|
||||
assert!(settings.smtp_settings.is_none());
|
||||
assert_eq!(settings.base_url.as_deref(), Some("https://x"));
|
||||
let map = settings.to_settings_map();
|
||||
assert_eq!(map["object_store_cache_config"], serde_json::Value::Null);
|
||||
assert_eq!(map["secret_backend"], serde_json::Value::Null);
|
||||
assert_eq!(map["smtp_settings"], serde_json::Value::Null);
|
||||
assert_eq!(map["base_url"], serde_json::json!("https://x"));
|
||||
}
|
||||
|
||||
/// Absent keys remain absent — critical so a PUT that only sets a single
|
||||
/// field doesn't accidentally delete every other setting in Merge mode.
|
||||
#[test]
|
||||
fn absent_typed_fields_do_not_appear_in_map() {
|
||||
let settings: GlobalSettings =
|
||||
serde_json::from_value(serde_json::json!({"base_url": "https://x"})).unwrap();
|
||||
let map = settings.to_settings_map();
|
||||
assert!(!map.contains_key("object_store_cache_config"));
|
||||
assert!(!map.contains_key("smtp_settings"));
|
||||
assert_eq!(map.get("base_url"), Some(&serde_json::json!("https://x")));
|
||||
}
|
||||
|
||||
/// End-to-end: deserialize → `to_settings_map` → diff in Merge mode with
|
||||
/// an explicit null produces a delete (the scenario that silently failed
|
||||
/// before the manual `Deserialize` impl).
|
||||
#[test]
|
||||
fn deserialize_then_diff_deletes_typed_null() {
|
||||
let mut current = BTreeMap::new();
|
||||
current.insert(
|
||||
"object_store_cache_config".to_string(),
|
||||
serde_json::json!({"type": "S3", "bucket": "b"}),
|
||||
);
|
||||
let desired: GlobalSettings =
|
||||
serde_json::from_value(serde_json::json!({"object_store_cache_config": null})).unwrap();
|
||||
let desired_map = desired.to_settings_map();
|
||||
let diff = diff_global_settings(¤t, &desired_map, ApplyMode::Merge);
|
||||
assert!(diff.upserts.is_empty());
|
||||
assert_eq!(diff.deletes, vec!["object_store_cache_config".to_string()]);
|
||||
}
|
||||
|
||||
/// Nested nulls inside a typed sub-struct are not promoted to top-level
|
||||
/// deletes — only the top-level key matters, mirroring the per-key API.
|
||||
#[test]
|
||||
fn nested_null_inside_typed_field_is_not_treated_as_top_level_null() {
|
||||
let settings: GlobalSettings =
|
||||
serde_json::from_value(serde_json::json!({"smtp_settings": {"smtp_host": null}}))
|
||||
.unwrap();
|
||||
let map = settings.to_settings_map();
|
||||
assert!(
|
||||
map["smtp_settings"].is_object(),
|
||||
"smtp_settings should be an object, not null"
|
||||
);
|
||||
}
|
||||
|
||||
/// Unknown/extra keys with explicit null still flow through `extra` — this
|
||||
/// was already correct before the manual impl; guard against regression.
|
||||
#[test]
|
||||
fn explicit_null_on_extra_field_survives_round_trip() {
|
||||
let settings: GlobalSettings =
|
||||
serde_json::from_value(serde_json::json!({"unknown_legacy_setting": null})).unwrap();
|
||||
let map = settings.to_settings_map();
|
||||
assert_eq!(map["unknown_legacy_setting"], serde_json::Value::Null);
|
||||
}
|
||||
|
||||
/// Top-level non-object input must reject with a deserialize error rather
|
||||
/// than silently producing defaults. Matches the previous derive behavior.
|
||||
#[test]
|
||||
fn non_object_top_level_input_errors() {
|
||||
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!(null)).is_err());
|
||||
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!("s")).is_err());
|
||||
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!(42)).is_err());
|
||||
assert!(serde_json::from_value::<GlobalSettings>(serde_json::json!([])).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn to_settings_map_empty_defaults() {
|
||||
let settings = GlobalSettings::default();
|
||||
|
||||
@@ -2,11 +2,11 @@
|
||||
//!
|
||||
//! Contains functions for converting Windmill schemas into MCP-compatible formats.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use serde_json::Value;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::types::SchemaType;
|
||||
use super::types::{ResourceInfo, ResourceType, SchemaType};
|
||||
use windmill_common::scripts::Schema;
|
||||
|
||||
/// Convert a Windmill Schema to a SchemaType
|
||||
@@ -22,38 +22,195 @@ pub fn convert_schema_to_schema_type(schema: Option<Schema>) -> SchemaType {
|
||||
schema_obj
|
||||
}
|
||||
|
||||
/// Extract resource type keys from a schema
|
||||
///
|
||||
/// Scans the schema properties for fields with format "resource-{type}"
|
||||
/// and returns a set of all unique resource type names found.
|
||||
pub fn extract_resource_types_from_schema(schema: &SchemaType) -> HashSet<String> {
|
||||
let mut resource_types = HashSet::new();
|
||||
for (_key, prop_value) in schema.properties.iter() {
|
||||
if let Value::Object(prop_map) = prop_value {
|
||||
if let Some(Value::String(format_str)) = prop_map.get("format") {
|
||||
if let Some(rt) = format_str.strip_prefix("resource-") {
|
||||
resource_types.insert(rt.to_string());
|
||||
}
|
||||
/// If `node` is a schema describing a Windmill resource reference, return the
|
||||
/// resource type name. Recognizes both on-disk shapes:
|
||||
/// - Form A (top-level scalar): `{ type: "object", format: "resource-<name>" }`
|
||||
/// - Form B (inside `items`): `{ type: "resource", resourceType: "<name>" }`
|
||||
fn resource_type_of_schema_node(node: &Value) -> Option<String> {
|
||||
let obj = node.as_object()?;
|
||||
if let Some(Value::String(fmt)) = obj.get("format") {
|
||||
if let Some(name) = fmt.strip_prefix("resource-") {
|
||||
return Some(name.to_string());
|
||||
}
|
||||
}
|
||||
if obj.get("type").and_then(Value::as_str) == Some("resource") {
|
||||
if let Some(Value::String(name)) = obj.get("resourceType") {
|
||||
return Some(name.clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Recursively collect resource type names referenced at any depth in `node`.
|
||||
fn collect_resource_types(node: &Value, out: &mut HashSet<String>) {
|
||||
if let Some(rt) = resource_type_of_schema_node(node) {
|
||||
out.insert(rt);
|
||||
}
|
||||
let Some(obj) = node.as_object() else { return };
|
||||
if let Some(Value::Object(props)) = obj.get("properties") {
|
||||
for v in props.values() {
|
||||
collect_resource_types(v, out);
|
||||
}
|
||||
}
|
||||
if let Some(items) = obj.get("items") {
|
||||
collect_resource_types(items, out);
|
||||
}
|
||||
if let Some(additional) = obj.get("additionalProperties") {
|
||||
if additional.is_object() {
|
||||
collect_resource_types(additional, out);
|
||||
}
|
||||
}
|
||||
for kw in ["allOf", "oneOf", "anyOf"] {
|
||||
if let Some(Value::Array(arr)) = obj.get(kw) {
|
||||
for s in arr {
|
||||
collect_resource_types(s, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract resource type keys referenced anywhere in a schema (top-level
|
||||
/// properties, nested objects, array items, additionalProperties, and
|
||||
/// allOf/oneOf/anyOf subschemas). Recognizes both the `format: resource-<name>`
|
||||
/// and `type: resource` + `resourceType` shapes.
|
||||
pub fn extract_resource_types_from_schema(schema: &SchemaType) -> HashSet<String> {
|
||||
let mut resource_types = HashSet::new();
|
||||
for prop_value in schema.properties.values() {
|
||||
collect_resource_types(prop_value, &mut resource_types);
|
||||
}
|
||||
resource_types
|
||||
}
|
||||
|
||||
/// Rewrite a single schema node that points to a Windmill resource: set
|
||||
/// `type: "string"` and inject a description listing the available resources.
|
||||
/// Mirrors the top-level behavior that used to live in
|
||||
/// `transform_schema_for_resources`. No-op if the resource type isn't in the
|
||||
/// pre-fetched cache.
|
||||
fn apply_resource_enrichment(
|
||||
prop_map: &mut Map<String, Value>,
|
||||
resource_type_key: &str,
|
||||
resources_cache: &HashMap<String, Vec<ResourceInfo>>,
|
||||
resources_types: &[ResourceType],
|
||||
) {
|
||||
let Some(resource_cache) = resources_cache.get(resource_type_key) else {
|
||||
return;
|
||||
};
|
||||
let resource_type = resources_types
|
||||
.iter()
|
||||
.find(|rt| rt.name == resource_type_key);
|
||||
let resources_count = resource_cache.len();
|
||||
let description = match resource_type {
|
||||
Some(rt) => format!(
|
||||
"This is a resource named `{}` with the following description: `{}`.\\nThe path of the resource should be used to specify the resource.\\n{}",
|
||||
rt.name,
|
||||
rt.description.as_deref().unwrap_or("No description"),
|
||||
if resources_count == 0 {
|
||||
"This resource does not have any available instances, you should create one from your windmill workspace."
|
||||
} else if resources_count > 1 {
|
||||
"This resource has multiple available instances, you should precisely select the one you want to use."
|
||||
} else {
|
||||
"There is 1 resource available."
|
||||
}
|
||||
),
|
||||
None => "An object parameter.".to_string(),
|
||||
};
|
||||
prop_map.insert("type".to_string(), Value::String("string".to_string()));
|
||||
prop_map.insert("description".to_string(), Value::String(description));
|
||||
// Drop the Windmill-internal keys we just consumed so the node is clean
|
||||
// regardless of whether `make_schema_compatible` runs after us. (Its strip
|
||||
// only fires while `type == "resource"`, which is no longer true here.)
|
||||
prop_map.remove("resourceType");
|
||||
if prop_map
|
||||
.get("format")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|s| s.starts_with("resource-"))
|
||||
{
|
||||
prop_map.remove("format");
|
||||
}
|
||||
if resources_count > 0 {
|
||||
let resources_description = resource_cache
|
||||
.iter()
|
||||
.map(|resource| {
|
||||
format!(
|
||||
"{}: $res:{}",
|
||||
resource.description.as_deref().unwrap_or("No title"),
|
||||
resource.path
|
||||
)
|
||||
})
|
||||
.collect::<Vec<String>>()
|
||||
.join("\\n");
|
||||
let prior_description = prop_map
|
||||
.get("description")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("No description")
|
||||
.to_string();
|
||||
prop_map.insert(
|
||||
"description".to_string(),
|
||||
Value::String(format!(
|
||||
"{}\\nHere are the available resources, in the format title:path. Title can be empty. Path should be used to specify the resource:\\n{}",
|
||||
prior_description, resources_description
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Walk a schema and enrich every Windmill-resource reference (in either shape,
|
||||
/// at any nesting depth) with `type: "string"` and a description listing
|
||||
/// available resources. The non-standard keys (`format: resource-*`,
|
||||
/// `resourceType`) consumed by the enrichment are stripped in place.
|
||||
pub fn enrich_resource_schemas(
|
||||
node: &mut Value,
|
||||
resources_cache: &HashMap<String, Vec<ResourceInfo>>,
|
||||
resources_types: &[ResourceType],
|
||||
) {
|
||||
if let Some(rt_key) = resource_type_of_schema_node(node) {
|
||||
if let Value::Object(obj) = node {
|
||||
apply_resource_enrichment(obj, &rt_key, resources_cache, resources_types);
|
||||
}
|
||||
}
|
||||
let Some(obj) = node.as_object_mut() else {
|
||||
return;
|
||||
};
|
||||
if let Some(Value::Object(props)) = obj.get_mut("properties") {
|
||||
for v in props.values_mut() {
|
||||
enrich_resource_schemas(v, resources_cache, resources_types);
|
||||
}
|
||||
}
|
||||
if let Some(items) = obj.get_mut("items") {
|
||||
enrich_resource_schemas(items, resources_cache, resources_types);
|
||||
}
|
||||
if let Some(additional) = obj.get_mut("additionalProperties") {
|
||||
if additional.is_object() {
|
||||
enrich_resource_schemas(additional, resources_cache, resources_types);
|
||||
}
|
||||
}
|
||||
for kw in ["allOf", "oneOf", "anyOf"] {
|
||||
if let Some(Value::Array(arr)) = obj.get_mut(kw) {
|
||||
for s in arr.iter_mut() {
|
||||
enrich_resource_schemas(s, resources_cache, resources_types);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Transform a JSON schema for maximum MCP client compatibility.
|
||||
///
|
||||
/// Ensures schemas conform to JSON Schema draft 2020-12 by:
|
||||
/// - Converting `integer` type to `number` (some clients don't support integer)
|
||||
/// - Removing invalid non-array `enum` values
|
||||
/// - Stripping non-standard keywords (`originalType`, `format` with `resource-*` prefix)
|
||||
/// - Rewriting the Windmill pseudo-type `type: "resource"` to `type: "string"`
|
||||
/// - Fixing contradictory schemas (`type: "string"` with `properties` → `type: "object"`)
|
||||
/// - Removing `default: null` when the type doesn't include `null`
|
||||
/// - Adding `type: "object"` to empty schemas that have no type
|
||||
pub fn make_schema_compatible(schema: &mut Value) {
|
||||
let Value::Object(obj) = schema else { return };
|
||||
|
||||
// 1. Strip non-standard keywords that aren't part of JSON Schema
|
||||
// 1. Strip non-standard keywords that aren't part of JSON Schema. The
|
||||
// Windmill-internal `resourceType` is dropped unconditionally so it can't
|
||||
// leak through even on enrichment cache-miss paths.
|
||||
obj.remove("originalType");
|
||||
obj.remove("resourceType");
|
||||
|
||||
// 2. Strip non-standard format values (resource-* is Windmill-internal)
|
||||
if obj
|
||||
@@ -64,6 +221,14 @@ pub fn make_schema_compatible(schema: &mut Value) {
|
||||
obj.remove("format");
|
||||
}
|
||||
|
||||
// 2b. Rewrite Windmill pseudo-type `resource` to `string`. The parser emits
|
||||
// this shape (with a sibling `resourceType` key) for `list[ResourceType]`
|
||||
// params; "resource" is not in the JSON Schema 2020-12 type enum and is
|
||||
// rejected by strict validators (e.g. Anthropic's tool registration).
|
||||
if obj.get("type").and_then(|v| v.as_str()) == Some("resource") {
|
||||
obj.insert("type".to_string(), Value::String("string".to_string()));
|
||||
}
|
||||
|
||||
// 3. Fix contradictory type: if `properties` is present, type must be "object"
|
||||
if obj.contains_key("properties") {
|
||||
match obj.get("type").and_then(|v| v.as_str()) {
|
||||
@@ -151,8 +316,27 @@ pub fn make_schema_compatible(schema: &mut Value) {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::make_schema_compatible;
|
||||
use super::*;
|
||||
use crate::common::types::{ResourceInfo, ResourceType};
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn aws_resources() -> (HashMap<String, Vec<ResourceInfo>>, Vec<ResourceType>) {
|
||||
let mut cache = HashMap::new();
|
||||
cache.insert(
|
||||
"c_aws_account".to_string(),
|
||||
vec![ResourceInfo {
|
||||
path: "f/platform/aws_dev".to_string(),
|
||||
description: Some("Dev account".to_string()),
|
||||
resource_type: "c_aws_account".to_string(),
|
||||
}],
|
||||
);
|
||||
let types = vec![ResourceType {
|
||||
name: "c_aws_account".to_string(),
|
||||
description: Some("AWS account".to_string()),
|
||||
}];
|
||||
(cache, types)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn converts_nested_integer_types() {
|
||||
@@ -356,6 +540,96 @@ mod tests {
|
||||
assert_eq!(schema, json!({}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_resource_pseudo_type_to_string() {
|
||||
let mut schema = json!({
|
||||
"type": "resource",
|
||||
"resourceType": "c_aws_account"
|
||||
});
|
||||
|
||||
make_schema_compatible(&mut schema);
|
||||
|
||||
assert_eq!(schema["type"], json!("string"));
|
||||
assert!(schema.get("resourceType").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_resource_pseudo_type_inside_array_items() {
|
||||
// Phocas repro: list[ResourceType] parameter. Anthropic rejected this
|
||||
// with "tools.<N>.custom.input_schema: JSON schema is invalid" because
|
||||
// "resource" is not in the draft 2020-12 type enum.
|
||||
let mut schema = json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"accounts": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "resource",
|
||||
"resourceType": "c_aws_account"
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
make_schema_compatible(&mut schema);
|
||||
|
||||
assert_eq!(
|
||||
schema["properties"]["accounts"]["items"]["type"],
|
||||
json!("string")
|
||||
);
|
||||
assert!(schema["properties"]["accounts"]["items"]
|
||||
.get("resourceType")
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrites_resource_pseudo_type_inside_nested_properties() {
|
||||
let mut schema = json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"db": {
|
||||
"type": "resource",
|
||||
"resourceType": "postgresql"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
make_schema_compatible(&mut schema);
|
||||
|
||||
assert_eq!(
|
||||
schema["properties"]["config"]["properties"]["db"]["type"],
|
||||
json!("string")
|
||||
);
|
||||
assert!(schema["properties"]["config"]["properties"]["db"]
|
||||
.get("resourceType")
|
||||
.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_nested_resource_format_inside_array_items() {
|
||||
let mut schema = json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dbs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"format": "resource-postgresql"
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
make_schema_compatible(&mut schema);
|
||||
|
||||
assert!(schema["properties"]["dbs"]["items"].get("format").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adds_type_to_schema_with_properties_but_no_type() {
|
||||
let mut schema = json!({
|
||||
@@ -368,4 +642,144 @@ mod tests {
|
||||
|
||||
assert_eq!(schema["type"], json!("object"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_resource_types_top_level_form_a() {
|
||||
let schema: SchemaType = serde_json::from_value(json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"db": { "type": "object", "format": "resource-postgresql" }
|
||||
},
|
||||
"required": []
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let types = extract_resource_types_from_schema(&schema);
|
||||
assert!(types.contains("postgresql"));
|
||||
assert_eq!(types.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_resource_types_inside_array_items_form_b() {
|
||||
let schema: SchemaType = serde_json::from_value(json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"accounts": {
|
||||
"type": "array",
|
||||
"items": { "type": "resource", "resourceType": "c_aws_account" }
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let types = extract_resource_types_from_schema(&schema);
|
||||
assert!(types.contains("c_aws_account"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extract_resource_types_inside_nested_properties_and_one_of() {
|
||||
let schema: SchemaType = serde_json::from_value(json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"config": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"db": { "type": "resource", "resourceType": "postgresql" }
|
||||
}
|
||||
},
|
||||
"either": {
|
||||
"oneOf": [
|
||||
{ "type": "object", "format": "resource-mysql" },
|
||||
{ "type": "string" }
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": []
|
||||
}))
|
||||
.unwrap();
|
||||
|
||||
let types = extract_resource_types_from_schema(&schema);
|
||||
assert!(types.contains("postgresql"));
|
||||
assert!(types.contains("mysql"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enrich_top_level_form_a_resource() {
|
||||
let (cache, types) = aws_resources();
|
||||
let mut node = json!({
|
||||
"type": "object",
|
||||
"format": "resource-c_aws_account"
|
||||
});
|
||||
|
||||
enrich_resource_schemas(&mut node, &cache, &types);
|
||||
|
||||
assert_eq!(node["type"], json!("string"));
|
||||
assert!(node.get("format").is_none());
|
||||
let desc = node["description"].as_str().unwrap();
|
||||
assert!(desc.contains("c_aws_account"));
|
||||
assert!(desc.contains("$res:f/platform/aws_dev"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enrich_inside_array_items_form_b() {
|
||||
// Phocas repro: items use the parser-style `type: resource` form.
|
||||
let (cache, types) = aws_resources();
|
||||
let mut node = json!({
|
||||
"type": "array",
|
||||
"items": { "type": "resource", "resourceType": "c_aws_account" }
|
||||
});
|
||||
|
||||
enrich_resource_schemas(&mut node, &cache, &types);
|
||||
|
||||
// The items schema should be rewritten to string with a description
|
||||
// listing the available resources, and the Windmill-internal
|
||||
// resourceType key should be stripped.
|
||||
assert_eq!(node["items"]["type"], json!("string"));
|
||||
assert!(node["items"].get("resourceType").is_none());
|
||||
let desc = node["items"]["description"].as_str().unwrap();
|
||||
assert!(desc.contains("$res:f/platform/aws_dev"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enrich_is_noop_when_resource_type_not_in_cache() {
|
||||
let mut node = json!({
|
||||
"type": "object",
|
||||
"format": "resource-unknown_type"
|
||||
});
|
||||
let before = node.clone();
|
||||
|
||||
enrich_resource_schemas(&mut node, &HashMap::new(), &[]);
|
||||
|
||||
assert_eq!(node, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enrich_deeply_nested_resource() {
|
||||
let (cache, types) = aws_resources();
|
||||
let mut node = json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"outer": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"inner": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "resource",
|
||||
"resourceType": "c_aws_account"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
enrich_resource_schemas(&mut node, &cache, &types);
|
||||
|
||||
assert_eq!(
|
||||
node["properties"]["outer"]["properties"]["inner"]["items"]["type"],
|
||||
json!("string")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +33,13 @@ pub trait McpAuth: Send + Sync + Clone + 'static {
|
||||
/// Get token scopes
|
||||
fn scopes(&self) -> Option<&[String]>;
|
||||
|
||||
/// True if the token was created with the `read_only` flag.
|
||||
/// When set, write-capable tools must be hidden from `list_tools` and
|
||||
/// rejected by `call_tool`. Defaults to false so existing impls compile.
|
||||
fn read_only(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Check if the user has an MCP scope
|
||||
fn has_mcp_scope(&self) -> bool {
|
||||
self.scopes()
|
||||
|
||||
@@ -26,6 +26,12 @@ pub struct EndpointTool {
|
||||
pub body_field_renames: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
/// True if this endpoint is safe to expose to a read-only token. Mirrors the
|
||||
/// `read_only_hint` computed by `create_endpoint_annotations`: only `GET`.
|
||||
pub fn is_endpoint_read_only(tool: &EndpointTool) -> bool {
|
||||
tool.method.as_ref() == "GET"
|
||||
}
|
||||
|
||||
/// Convert a single endpoint tool to MCP tool
|
||||
pub fn endpoint_tool_to_mcp_tool(tool: &EndpointTool) -> Tool {
|
||||
let mut combined_properties = serde_json::Map::new();
|
||||
|
||||
@@ -12,7 +12,7 @@ pub mod tools;
|
||||
|
||||
// Re-export main types
|
||||
pub use backend::{BackendResult, McpAuth, McpBackend};
|
||||
pub use endpoints::{endpoint_tool_to_mcp_tool, EndpointTool};
|
||||
pub use endpoints::{endpoint_tool_to_mcp_tool, is_endpoint_read_only, EndpointTool};
|
||||
pub use runner::Runner;
|
||||
pub use tools::create_tool_from_item;
|
||||
|
||||
|
||||
@@ -145,94 +145,100 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
|
||||
|
||||
let favorites_only = scope_config.favorites;
|
||||
|
||||
// Fetch all items concurrently
|
||||
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
|
||||
self.backend
|
||||
.list_scripts(&auth, &workspace_id, favorites_only, None),
|
||||
self.backend
|
||||
.list_flows(&auth, &workspace_id, favorites_only, None),
|
||||
self.backend.list_resource_types(&auth, &workspace_id),
|
||||
async {
|
||||
if let Some(ref apps) = scope_config.hub_apps {
|
||||
self.backend.list_hub_scripts(Some(apps)).await
|
||||
} else {
|
||||
Ok(vec![])
|
||||
}
|
||||
}
|
||||
)?;
|
||||
|
||||
// Filter items based on scope
|
||||
let filtered_scripts: Vec<_> = scripts
|
||||
.into_iter()
|
||||
.filter(|s| !scope_config.granular || scope_config.is_allowed("script", &s.path))
|
||||
.collect();
|
||||
|
||||
let filtered_flows: Vec<_> = flows
|
||||
.into_iter()
|
||||
.filter(|f| !scope_config.granular || scope_config.is_allowed("flow", &f.path))
|
||||
.collect();
|
||||
|
||||
// Collect all needed resource types from all schemas
|
||||
let mut needed_resource_types: HashSet<String> = HashSet::new();
|
||||
for script in &filtered_scripts {
|
||||
needed_resource_types.extend(extract_resource_types_from_schema(&script.get_schema()));
|
||||
}
|
||||
for flow in &filtered_flows {
|
||||
needed_resource_types.extend(extract_resource_types_from_schema(&flow.get_schema()));
|
||||
}
|
||||
for hub_script in &hub_scripts {
|
||||
needed_resource_types
|
||||
.extend(extract_resource_types_from_schema(&hub_script.get_schema()));
|
||||
}
|
||||
|
||||
// Pre-fetch all resources
|
||||
let resource_futures: Vec<_> = needed_resource_types
|
||||
.into_iter()
|
||||
.map(|rt| {
|
||||
let backend = self.backend.clone();
|
||||
let auth = auth.clone();
|
||||
let workspace_id = workspace_id.clone();
|
||||
async move {
|
||||
backend
|
||||
.list_resources(&auth, &workspace_id, &rt)
|
||||
.await
|
||||
.map(|resources| (rt, resources))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let resource_results = futures::future::try_join_all(resource_futures).await?;
|
||||
let resources_cache: HashMap<String, Vec<ResourceInfo>> =
|
||||
resource_results.into_iter().collect();
|
||||
let read_only = auth.read_only();
|
||||
|
||||
let mut tools = Vec::new();
|
||||
|
||||
for script in &filtered_scripts {
|
||||
tools.push(create_tool_from_item(
|
||||
script,
|
||||
self.backend.as_ref(),
|
||||
&resources_cache,
|
||||
&resource_types,
|
||||
));
|
||||
}
|
||||
// Read-only tokens cannot run scripts/flows/hub-scripts (running is a
|
||||
// mutating action), so skip the script/flow/hub/resource fetches
|
||||
// entirely — they would only be discarded below.
|
||||
if !read_only {
|
||||
let (scripts, flows, resource_types, hub_scripts) = tokio::try_join!(
|
||||
self.backend
|
||||
.list_scripts(&auth, &workspace_id, favorites_only, None),
|
||||
self.backend
|
||||
.list_flows(&auth, &workspace_id, favorites_only, None),
|
||||
self.backend.list_resource_types(&auth, &workspace_id),
|
||||
async {
|
||||
if let Some(ref apps) = scope_config.hub_apps {
|
||||
self.backend.list_hub_scripts(Some(apps)).await
|
||||
} else {
|
||||
Ok(vec![])
|
||||
}
|
||||
}
|
||||
)?;
|
||||
|
||||
for flow in &filtered_flows {
|
||||
tools.push(create_tool_from_item(
|
||||
flow,
|
||||
self.backend.as_ref(),
|
||||
&resources_cache,
|
||||
&resource_types,
|
||||
));
|
||||
}
|
||||
let filtered_scripts: Vec<_> = scripts
|
||||
.into_iter()
|
||||
.filter(|s| !scope_config.granular || scope_config.is_allowed("script", &s.path))
|
||||
.collect();
|
||||
|
||||
for hub_script in &hub_scripts {
|
||||
tools.push(create_tool_from_item(
|
||||
hub_script,
|
||||
self.backend.as_ref(),
|
||||
&resources_cache,
|
||||
&resource_types,
|
||||
));
|
||||
let filtered_flows: Vec<_> = flows
|
||||
.into_iter()
|
||||
.filter(|f| !scope_config.granular || scope_config.is_allowed("flow", &f.path))
|
||||
.collect();
|
||||
|
||||
// Collect all needed resource types from all schemas
|
||||
let mut needed_resource_types: HashSet<String> = HashSet::new();
|
||||
for script in &filtered_scripts {
|
||||
needed_resource_types
|
||||
.extend(extract_resource_types_from_schema(&script.get_schema()));
|
||||
}
|
||||
for flow in &filtered_flows {
|
||||
needed_resource_types
|
||||
.extend(extract_resource_types_from_schema(&flow.get_schema()));
|
||||
}
|
||||
for hub_script in &hub_scripts {
|
||||
needed_resource_types
|
||||
.extend(extract_resource_types_from_schema(&hub_script.get_schema()));
|
||||
}
|
||||
|
||||
// Pre-fetch all resources
|
||||
let resource_futures: Vec<_> = needed_resource_types
|
||||
.into_iter()
|
||||
.map(|rt| {
|
||||
let backend = self.backend.clone();
|
||||
let auth = auth.clone();
|
||||
let workspace_id = workspace_id.clone();
|
||||
async move {
|
||||
backend
|
||||
.list_resources(&auth, &workspace_id, &rt)
|
||||
.await
|
||||
.map(|resources| (rt, resources))
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let resource_results = futures::future::try_join_all(resource_futures).await?;
|
||||
let resources_cache: HashMap<String, Vec<ResourceInfo>> =
|
||||
resource_results.into_iter().collect();
|
||||
|
||||
for script in &filtered_scripts {
|
||||
tools.push(create_tool_from_item(
|
||||
script,
|
||||
self.backend.as_ref(),
|
||||
&resources_cache,
|
||||
&resource_types,
|
||||
));
|
||||
}
|
||||
|
||||
for flow in &filtered_flows {
|
||||
tools.push(create_tool_from_item(
|
||||
flow,
|
||||
self.backend.as_ref(),
|
||||
&resources_cache,
|
||||
&resource_types,
|
||||
));
|
||||
}
|
||||
|
||||
for hub_script in &hub_scripts {
|
||||
tools.push(create_tool_from_item(
|
||||
hub_script,
|
||||
self.backend.as_ref(),
|
||||
&resources_cache,
|
||||
&resource_types,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Add endpoint tools from the generated MCP tools, filtered by scope
|
||||
@@ -241,6 +247,9 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
if scope_config.granular && !scope_config.is_allowed("endpoint", &endpoint_tool.name) {
|
||||
continue;
|
||||
}
|
||||
if read_only && !crate::server::is_endpoint_read_only(&endpoint_tool) {
|
||||
continue;
|
||||
}
|
||||
|
||||
tools.push(endpoint_tool_to_mcp_tool(&endpoint_tool));
|
||||
}
|
||||
@@ -259,6 +268,7 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
let scopes = auth.scopes().unwrap_or(&[]);
|
||||
let scope_config =
|
||||
parse_mcp_scopes(scopes).map_err(|e| ErrorData::internal_error(e, None))?;
|
||||
let read_only = auth.read_only();
|
||||
|
||||
let args = request.arguments.map(Value::Object).unwrap_or(Value::Null);
|
||||
|
||||
@@ -278,6 +288,15 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
None,
|
||||
));
|
||||
}
|
||||
if read_only && !crate::server::is_endpoint_read_only(endpoint_tool) {
|
||||
return Err(ErrorData::internal_error(
|
||||
format!(
|
||||
"Access denied: endpoint '{}' is not read-only and this token is restricted to read-only operations",
|
||||
endpoint_tool.name
|
||||
),
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
// This is an endpoint tool, call via backend
|
||||
let result = self
|
||||
@@ -294,6 +313,18 @@ impl<B: McpBackend> ServerHandler for Runner<B> {
|
||||
}
|
||||
}
|
||||
|
||||
// Anything below this point runs a script or flow, which is a mutating
|
||||
// action and must be denied for read-only tokens.
|
||||
if read_only {
|
||||
return Err(ErrorData::internal_error(
|
||||
format!(
|
||||
"Access denied: tool '{}' runs a script/flow and this token is restricted to read-only operations",
|
||||
request.name
|
||||
),
|
||||
None,
|
||||
));
|
||||
}
|
||||
|
||||
// Resolve the tool name to (type, path, is_hub)
|
||||
let (type_str, is_hub, is_hashed) = parse_tool_prefix(&request.name).map_err(|e| {
|
||||
ErrorData::internal_error(format!("Failed to parse tool name: {}", e), None)
|
||||
|
||||
@@ -94,6 +94,7 @@ async fn new_webhook_token(
|
||||
None,
|
||||
Some(scopes),
|
||||
Some(workspace_id.to_owned()),
|
||||
None,
|
||||
);
|
||||
let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?;
|
||||
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
//! End-to-end coverage for the EE HTTP-tracing path on nativets.
|
||||
//!
|
||||
//! Pairs with `otel_init.rs` (which pins only the `OTEL_GLOBALS`
|
||||
//! population contract). This test exercises the full chain that
|
||||
//! actually delivers a span to a collector:
|
||||
//!
|
||||
//! `deno_telemetry::init` with EE config (Rust)
|
||||
//! → `globalThis.__bootstrapOtel()` (JS, flips TRACING_ENABLED)
|
||||
//! → user `fetch()` → deno_fetch's `builtinTracer().startSpan`
|
||||
//! → `BatchSpanProcessor` → `HttpExporter` (OTLP/HTTP-binary)
|
||||
//! → our mock OTLP listener captures the request bytes
|
||||
//!
|
||||
//! Without the v1.702.0 fix (#573 EE / #9163 OSS), the third arrow
|
||||
//! panics in a tokio worker. With the fix in place, the span is
|
||||
//! emitted and shows up at the listener — which is what the customer
|
||||
//! is paying for when they enable HTTP tracing on nativets.
|
||||
//!
|
||||
//! `#[ignore]`'d: spins a V8 isolate (~seconds) and binds two TCP
|
||||
//! listeners. Run with `cargo test -p windmill-runtime-nativets
|
||||
//! --test otel_e2e -- --ignored`.
|
||||
//!
|
||||
//! Owns its own test binary so `OTEL_GLOBALS`'s `OnceCell` doesn't
|
||||
//! race with `otel_init.rs`.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use windmill_runtime_nativets::{deno_telemetry, transpile_ts, NativeAnnotation, PrewarmedIsolate};
|
||||
|
||||
/// Bind 127.0.0.1:0 and spawn an accept loop. Each connection is
|
||||
/// read until idle/EOF, the body is appended to `captured`, then we
|
||||
/// respond with HTTP 200. Returns the bound port.
|
||||
async fn spawn_capturing_http(captured: Arc<Mutex<Vec<Vec<u8>>>>) -> u16 {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let Ok((mut sock, _)) = listener.accept().await else {
|
||||
break;
|
||||
};
|
||||
let captured = captured.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut buf = Vec::new();
|
||||
let mut tmp = [0u8; 8192];
|
||||
let _ = tokio::time::timeout(Duration::from_millis(300), async {
|
||||
loop {
|
||||
match sock.read(&mut tmp).await {
|
||||
Ok(0) | Err(_) => break,
|
||||
Ok(n) => buf.extend_from_slice(&tmp[..n]),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let _ = sock
|
||||
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n")
|
||||
.await;
|
||||
captured.lock().await.push(buf);
|
||||
});
|
||||
}
|
||||
});
|
||||
port
|
||||
}
|
||||
|
||||
/// Initialize `deno_telemetry` with the exact `OtelConfig` shape that
|
||||
/// the EE `load_internal_otel_exporter` ships in production. Keeps
|
||||
/// this test in lockstep with the actual call site: if production
|
||||
/// drifts away from `tracing_enabled + Capture`, this test breaks
|
||||
/// before the customer-facing panic does.
|
||||
fn init_with_ee_otel_config() {
|
||||
deno_telemetry::init(
|
||||
deno_telemetry::OtelRuntimeConfig {
|
||||
runtime_name: "windmill-nativets".into(),
|
||||
runtime_version: "0".into(),
|
||||
},
|
||||
deno_telemetry::OtelConfig {
|
||||
tracing_enabled: true,
|
||||
console: deno_telemetry::OtelConsoleConfig::Capture,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("deno_telemetry init failed");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[ignore = "spins V8 + tcp listeners; run with --ignored"]
|
||||
async fn fetch_after_init_otel_emits_span_to_collector() {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
|
||||
// 1. Stand up two listeners: one that pretends to be the user's
|
||||
// fetch target, one that pretends to be the OTLP collector.
|
||||
let fetch_hits: Arc<Mutex<Vec<Vec<u8>>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let otlp_hits: Arc<Mutex<Vec<Vec<u8>>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let fetch_port = spawn_capturing_http(fetch_hits.clone()).await;
|
||||
let otlp_port = spawn_capturing_http(otlp_hits.clone()).await;
|
||||
|
||||
// 2. Point the deno_telemetry exporter at the mock collector and
|
||||
// initialize. Mirrors `load_internal_otel_exporter` in EE.
|
||||
//
|
||||
// SAFETY: test runs in its own process binary; no other thread
|
||||
// reads OTEL_EXPORTER_OTLP_ENDPOINT before init returns.
|
||||
unsafe {
|
||||
std::env::set_var(
|
||||
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
||||
format!("http://127.0.0.1:{otlp_port}"),
|
||||
);
|
||||
}
|
||||
init_with_ee_otel_config();
|
||||
assert!(
|
||||
deno_telemetry::OTEL_GLOBALS.get().is_some(),
|
||||
"OTEL_GLOBALS missing — load_internal_otel_exporter's config regressed?"
|
||||
);
|
||||
|
||||
// 3. Run user TS that bootstraps OTel and then issues a fetch
|
||||
// at the mock target. `__bootstrapOtel` is fire-and-forget
|
||||
// (resolves a dynamic import on the microtask queue); the
|
||||
// `setTimeout` loop yields a few times so the import resolves
|
||||
// and the `TRACING_ENABLED` flag is set before fetch runs.
|
||||
let ts = format!(
|
||||
r#"
|
||||
declare const globalThis: any;
|
||||
export async function main(): Promise<number> {{
|
||||
globalThis.__bootstrapOtel();
|
||||
// Yield multiple microtask + timer turns so the dynamic import
|
||||
// in __bootstrapOtel resolves and TRACING_ENABLED flips before
|
||||
// fetch runs (otherwise deno_fetch skips the span entirely).
|
||||
for (let i = 0; i < 5; i++) {{
|
||||
await new Promise<void>(r => setTimeout(r, 10));
|
||||
}}
|
||||
const resp = await fetch("http://127.0.0.1:{fetch_port}/probe");
|
||||
return resp.status;
|
||||
}}
|
||||
"#
|
||||
);
|
||||
let js = transpile_ts(ts).expect("transpile failed");
|
||||
let ann = NativeAnnotation { useragent: None, proxy: None };
|
||||
|
||||
let mut iso = PrewarmedIsolate::spawn(String::new(), js, ann, vec![], None);
|
||||
iso.wait_ready().await.expect("isolate failed to pre-warm");
|
||||
let res = iso
|
||||
.start_execution("{}".to_string())
|
||||
.wait()
|
||||
.await
|
||||
.expect("isolate panicked");
|
||||
|
||||
// The exact bug: pre-fix, fetch panics this isolate. Post-fix,
|
||||
// we get back the mock target's 200.
|
||||
let raw = res.result.expect("user script returned an error");
|
||||
assert_eq!(raw.get(), "200", "fetch should return mock target status");
|
||||
|
||||
assert_eq!(
|
||||
fetch_hits.lock().await.len(),
|
||||
1,
|
||||
"fetch target should have been hit exactly once"
|
||||
);
|
||||
|
||||
// 4. Force the BatchSpanProcessor to flush so the exporter posts
|
||||
// to our mock collector synchronously (default flush interval
|
||||
// is ~5s; tests can't wait that long).
|
||||
deno_telemetry::flush();
|
||||
// Exporter is async over the OTel runtime; give it a beat to
|
||||
// actually send the HTTP request.
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
let otlp_captured = otlp_hits.lock().await;
|
||||
assert!(
|
||||
!otlp_captured.is_empty(),
|
||||
"OTLP collector should have received at least one export — \
|
||||
spans aren't reaching the collector after init"
|
||||
);
|
||||
|
||||
// Verify the export carries our fetch span. OTLP is protobuf, so
|
||||
// grep the raw bytes for OTel HTTP semantic-convention markers
|
||||
// that deno_fetch's auto-instrumentation attaches:
|
||||
// - the target URL ("url.full" attribute)
|
||||
// - "http.request.method" attribute
|
||||
let combined: Vec<u8> = otlp_captured.iter().flatten().copied().collect();
|
||||
let bytes_contain = |needle: &[u8]| combined.windows(needle.len()).any(|w| w == needle);
|
||||
|
||||
let url_marker = format!("http://127.0.0.1:{fetch_port}/probe");
|
||||
let has_url = bytes_contain(url_marker.as_bytes());
|
||||
let has_method_attr = bytes_contain(b"http.request.method");
|
||||
|
||||
if !(has_url && has_method_attr) {
|
||||
eprintln!(
|
||||
"OTLP bytes ({}): {:?}",
|
||||
combined.len(),
|
||||
String::from_utf8_lossy(&combined)
|
||||
);
|
||||
}
|
||||
|
||||
assert!(
|
||||
has_url,
|
||||
"exported OTLP body should reference the fetched URL ({}); got {} bytes",
|
||||
url_marker,
|
||||
combined.len()
|
||||
);
|
||||
assert!(
|
||||
has_method_attr,
|
||||
"exported OTLP body should carry HTTP semconv attributes (http.request.method); got {} bytes",
|
||||
combined.len()
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
//! Regression test for the deno_telemetry 0.31 nativets-fetch panic.
|
||||
//!
|
||||
//! Setup: when EE's `load_internal_otel_exporter` runs (HTTP tracing
|
||||
//! enabled), it calls `deno_telemetry::init` and then flips
|
||||
//! `DENO_OTEL_INITIALIZED=true` so `js_eval` will run
|
||||
//! `globalThis.__bootstrapOtel()` inside the nativets JsRuntime. That
|
||||
//! bootstrap unconditionally sets JS-side `TRACING_ENABLED=true`, which
|
||||
//! makes `deno_fetch`'s `fetch()` call `builtinTracer().startSpan(...)`
|
||||
//! — and `OtelTracer::builtin()` does `OTEL_GLOBALS.get().unwrap()`.
|
||||
//!
|
||||
//! In deno_telemetry 0.31, `init` was given an early-return guard: if
|
||||
//! `tracing_enabled`, `metrics_enabled`, and `console` are all
|
||||
//! off/Ignore, it returns `Ok(())` *without populating OTEL_GLOBALS*.
|
||||
//! v1.700.0 was passing `OtelConfig::default()` (all off) — so the JS
|
||||
//! bootstrap proceeded but the Rust-side OnceCell was empty, and the
|
||||
//! first `fetch()` panicked the tokio worker in a context that cannot
|
||||
//! unwind. Fixed in v1.702.0 by passing `tracing_enabled: true` +
|
||||
//! `console: Capture` from `load_internal_otel_exporter` (PRs #573 EE
|
||||
//! / #9163 OSS), matching the JS bootstrap shape `[1, 0, 1, 0]`.
|
||||
//!
|
||||
//! This test pins that contract directly against `deno_telemetry::
|
||||
//! init` — the function the EE call site invokes — so the next time
|
||||
//! the dep is bumped and someone is tempted to "simplify" the config
|
||||
//! back to `OtelConfig::default()`, CI catches it.
|
||||
//!
|
||||
//! Lives in `tests/` (not `src/`) so it gets its own test binary and
|
||||
//! its own process — `OTEL_GLOBALS` is a `OnceCell`, so sharing it
|
||||
//! with the smoke tests in `src/smoke_tests.rs` would race.
|
||||
|
||||
use windmill_runtime_nativets::deno_telemetry;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn ee_otel_init_config_populates_otel_globals() {
|
||||
// deno_telemetry::init builds an HttpExporter (uses rustls) — it
|
||||
// needs a process-wide CryptoProvider, which the real binary
|
||||
// installs in setup_deno_runtime / main. Mirror that here.
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
|
||||
// Pre-condition: nothing else in this test binary has touched the
|
||||
// OnceCell, so it must be empty.
|
||||
assert!(
|
||||
deno_telemetry::OTEL_GLOBALS.get().is_none(),
|
||||
"OTEL_GLOBALS must start empty in a fresh test process"
|
||||
);
|
||||
|
||||
// Step 1: reproduce the footgun. `OtelConfig::default()` has
|
||||
// tracing/metrics off and `console = Ignore`, which trips the
|
||||
// 0.31 early-return — Ok(()) but OnceCell stays empty.
|
||||
deno_telemetry::init(
|
||||
deno_telemetry::OtelRuntimeConfig {
|
||||
runtime_name: "windmill-nativets-test".into(),
|
||||
runtime_version: "0".into(),
|
||||
},
|
||||
deno_telemetry::OtelConfig::default(),
|
||||
)
|
||||
.expect("init with default config returns Ok (early-return)");
|
||||
|
||||
assert!(
|
||||
deno_telemetry::OTEL_GLOBALS.get().is_none(),
|
||||
"deno_telemetry 0.31 contract: init with all-disabled config \
|
||||
must NOT populate OTEL_GLOBALS — if this changes on a future \
|
||||
bump, load_internal_otel_exporter can drop the explicit \
|
||||
tracing_enabled/console fields."
|
||||
);
|
||||
|
||||
// Step 2: re-call init with the exact config shape that the EE
|
||||
// `load_internal_otel_exporter` ships — this is what production
|
||||
// hits, so the test exercises the actual prod call path.
|
||||
deno_telemetry::init(
|
||||
deno_telemetry::OtelRuntimeConfig {
|
||||
runtime_name: "windmill-nativets".into(),
|
||||
runtime_version: "0".into(),
|
||||
},
|
||||
deno_telemetry::OtelConfig {
|
||||
tracing_enabled: true,
|
||||
console: deno_telemetry::OtelConsoleConfig::Capture,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("init with tracing_enabled config must succeed on a fresh OnceCell");
|
||||
|
||||
assert!(
|
||||
deno_telemetry::OTEL_GLOBALS.get().is_some(),
|
||||
"load_internal_otel_exporter's OtelConfig must populate \
|
||||
OTEL_GLOBALS so OtelTracer::builtin() does not panic on \
|
||||
.unwrap() once __bootstrapOtel flips JS-side TRACING_ENABLED"
|
||||
);
|
||||
}
|
||||
@@ -39,7 +39,7 @@ ruby = ["dep:windmill-parser-ruby"]
|
||||
rlang = ["dep:windmill-parser-r"]
|
||||
duckdb = ["dep:libloading"]
|
||||
quickjs = ["windmill-jseval/quickjs"]
|
||||
bedrock = ["windmill-ai/bedrock", "dep:aws-sdk-bedrockruntime", "dep:aws-config", "dep:aws-credential-types", "dep:aws-smithy-types"]
|
||||
bedrock = ["windmill-ai/bedrock"]
|
||||
|
||||
[dependencies]
|
||||
windmill-ai = { workspace = true, default-features = false }
|
||||
@@ -71,10 +71,6 @@ windmill-parser-sql.workspace = true
|
||||
windmill-parser-graphql.workspace = true
|
||||
windmill-parser-php = { workspace = true, optional = true }
|
||||
windmill-git-sync.workspace = true
|
||||
aws-sdk-bedrockruntime = { workspace = true, optional = true }
|
||||
aws-config = { workspace = true, optional = true }
|
||||
aws-credential-types = { workspace = true, optional = true }
|
||||
aws-smithy-types = { workspace = true, optional = true }
|
||||
flume.workspace = true
|
||||
sqlx.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// AI executor module structure
|
||||
// This module will contain all AI-related execution logic
|
||||
|
||||
pub mod providers;
|
||||
pub mod query_builder;
|
||||
pub mod tools;
|
||||
pub mod utils;
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user