starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-08 16:03:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'bench/sim-k8s-util-panel' of github.com:windmill-labs/windmill into bench/sim-k8s-util-panel

Browse Source
This commit is contained in:
pyranota
2026-06-08 12:06:11 +02:00
parent c6569a3a63 31a1e7f992
commit c94bf9cef0
454 changed files with 38951 additions and 10510 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/DockerfileBackendTests
+1 -1
View File
@@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
# UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
ENV TZ=Etc/UTC
.github/change-versions-mac.sh
+1 -1
View File
@@ -7,7 +7,7 @@ VERSION=$1
echo "Updating versions to: $VERSION"
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/cli/src/main.ts
sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/cli/src/core/constants.ts
sed -i '' -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
sed -i '' -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml
.github/change-versions.sh
+1 -1
View File
@@ -7,7 +7,7 @@ VERSION=$1
echo "Updating versions to: $VERSION"
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/backend/Cargo.toml
sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/src/main.ts
sed -i -e "/^export const VERSION =/s/= .*/= \"$VERSION\";/" ${root_dirpath}/cli/src/core/constants.ts
sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" ${root_dirpath}/benchmarks/lib.ts
sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/backend/windmill-api/openapi.yaml
sed -i -e "/version: /s/: .*/: $VERSION/" ${root_dirpath}/openflow.openapi.yaml
.github/workflows/backend-test-windows.yml
+32 -6
View File
@@ -74,7 +74,7 @@ jobs:
- uses: astral-sh/setup-uv@v6.2.1
with:
version: "0.9.24"
version: "0.9.25"
- uses: shivammathur/setup-php@v2
with:
@@ -98,6 +98,21 @@ jobs:
vcpkg.exe install openssl:x64-windows-static
vcpkg.exe integrate install
- name: Free disk space (post-vcpkg)
shell: pwsh
run: |
# vcpkg leaves multi-GB of buildtrees/downloads after installing openssl;
# we only need the installed/ dir for linking.
$vcpkgRoot = $env:VCPKG_INSTALLATION_ROOT
foreach ($sub in @("buildtrees", "downloads", "packages")) {
$path = Join-Path $vcpkgRoot $sub
if (Test-Path $path) {
Write-Host "Removing $path"
Remove-Item -Recurse -Force -ErrorAction SilentlyContinue $path
}
}
Get-PSDrive C | Select-Object Used,Free | Format-Table -AutoSize
- name: Get runtime paths
id: runtime-paths
shell: pwsh
@@ -119,6 +134,10 @@ jobs:
cargo build --release -p windmill_duckdb_ffi_internal
New-Item -ItemType Directory -Path ..\target\debug -Force
Copy-Item target\release\windmill_duckdb_ffi_internal.dll ..\target\debug\
# duckdb is bundled (~2GB of build artifacts); the DLL is the only
# thing we need from this excluded-crate target dir.
Remove-Item -Recurse -Force -ErrorAction SilentlyContinue target
Get-PSDrive C | Select-Object Used,Free | Format-Table -AutoSize
- name: Print runtime versions and env
shell: pwsh
@@ -136,6 +155,10 @@ jobs:
echo "USERPROFILE=$env:USERPROFILE"
echo "HOME=$env:HOME"
- name: Disk space before cargo test
shell: pwsh
run: Get-PSDrive C | Select-Object Used,Free | Format-Table -AutoSize
- name: cargo test
working-directory: backend
timeout-minutes: 60
@@ -144,13 +167,16 @@ jobs:
RUST_LOG: "off"
RUST_LOG_STYLE: never
CARGO_NET_GIT_FETCH_WITH_CLI: true
CARGO_BUILD_JOBS: 12
# 16-vcpu runners with disabled PDB still hit LNK1180 ("insufficient
# disk space") at link time with 12 parallel link jobs: each test
# binary link spikes several hundred MB of transient I/O. Capping at
# 8 trades ~25% wall time for headroom on the ~75GB runner disk.
CARGO_BUILD_JOBS: 8
# backend/Cargo.toml sets split-debuginfo = "unpacked", which on
# windows-msvc is coerced to "packed": every test-binary link spawns
# the mspdbsrv.exe PDB type server and writes a large .pdb. With 12
# parallel link jobs this races the type-server cap (LNK1318 "LIMIT
# (12)") and exhausts the runner disk (LNK1180). CI needs no debug
# info, so disable PDB generation for the dev/test profiles here.
# the mspdbsrv.exe PDB type server and writes a large .pdb. CI needs
# no debug info, so disable PDB generation for the dev/test profiles
# here (avoids both LNK1318 type-server limit and PDB disk usage).
CARGO_PROFILE_DEV_SPLIT_DEBUGINFO: "off"
CARGO_PROFILE_TEST_SPLIT_DEBUGINFO: "off"
# Tests' poll-time stack frames (deep nested async fn chains in
.github/workflows/backend-test.yml
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
node-version: "20"
- uses: astral-sh/setup-uv@v6.2.1
with:
version: "0.9.24"
version: "0.9.25"
- uses: shivammathur/setup-php@v2
with:
php-version: "8.3"
.github/workflows/check-empty-fixture.yml
+19
View File
@@ -0,0 +1,19 @@
name: Check fixture is empty
on:
push:
branches: [main]
paths:
- "fixtures/**"
pull_request:
paths:
- "fixtures/**"
jobs:
check-empty-fixture:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Ensure fixtures/cli-sync/ has no committed snapshot
run: bash fixtures/check-empty.sh
.github/workflows/git-sync-test.yml
+3 -1
View File
@@ -8,6 +8,7 @@ on:
- "backend/windmill-git-sync/**"
- "backend/windmill-api-integration-tests/tests/git_sync*"
- "backend/ee-repo-ref.txt"
- "backend/windmill-common/src/workspaces.rs"
- "integration_tests/test/git_sync_test.py"
- ".github/workflows/git-sync-test.yml"
pull_request:
@@ -16,6 +17,7 @@ on:
- "backend/windmill-git-sync/**"
- "backend/windmill-api-integration-tests/tests/git_sync*"
- "backend/ee-repo-ref.txt"
- "backend/windmill-common/src/workspaces.rs"
- "integration_tests/test/git_sync_test.py"
- ".github/workflows/git-sync-test.yml"
@@ -49,7 +51,7 @@ jobs:
echo "$CHANGED_FILES"
# Direct git sync file changes — always relevant
if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then
if echo "$CHANGED_FILES" | grep -qE '^(backend/windmill-git-sync/|backend/windmill-api-integration-tests/tests/git_sync|backend/windmill-common/src/workspaces\.rs|integration_tests/test/git_sync|\.github/workflows/git-sync-test\.yml)'; then
echo "should_run=true" >> "$GITHUB_OUTPUT"
echo "Relevant: direct git sync file changes"
exit 0
AGENTS.md
+1
View File
@@ -106,3 +106,4 @@ $NAV --root backend callees "X" # what does X call?
- Search for existing code to reuse before writing new code
- Follow established patterns in the codebase
- Keep changes focused — don't refactor beyond what's asked
- **Never attribute work to a specific customer, account, or "requested by a customer" in repo-tracked content** (PR descriptions, commit messages, code comments, docs). Describe changes by their technical motivation instead.
CHANGELOG.md
+217
View File
@@ -1,5 +1,222 @@
# Changelog
## [1.719.0](https://github.com/windmill-labs/windmill/compare/v1.718.0...v1.719.0) (2026-06-06)
### Features
* **otel:** connect jobs to the inbound distributed trace ([#9456](https://github.com/windmill-labs/windmill/issues/9456)) ([fad1a54](https://github.com/windmill-labs/windmill/commit/fad1a549d95c00d0746a48163c4f95fc69733e1a))
### Bug Fixes
* authenticate slack callback payload with per-workspace hmac ([#9461](https://github.com/windmill-labs/windmill/issues/9461)) ([fbdf81b](https://github.com/windmill-labs/windmill/commit/fbdf81ba5f77d282c025360ecee14138dd4cb4a2))
* prevent token label collision bypassing job read access control ([#9462](https://github.com/windmill-labs/windmill/issues/9462)) ([e1e7af6](https://github.com/windmill-labs/windmill/commit/e1e7af6a25a44eb06b67332ce1efeae2a21e0c6d))
* **python:** escape reserved-keyword step ids in wrapper codegen ([#9460](https://github.com/windmill-labs/windmill/issues/9460)) ([6a15a9b](https://github.com/windmill-labs/windmill/commit/6a15a9b152ad20be4b5c3de6000516da231e41e0)), closes [#8893](https://github.com/windmill-labs/windmill/issues/8893)
## [1.718.0](https://github.com/windmill-labs/windmill/compare/v1.717.1...v1.718.0) (2026-06-05)
### Features
* **flows:** opt-in to include the stopping step's result in early-stop errors ([#9446](https://github.com/windmill-labs/windmill/issues/9446)) ([f2f0812](https://github.com/windmill-labs/windmill/commit/f2f0812a04c9256cfc8eba5e0dcf38d71d971410))
* make C# dotnet target framework configurable via DOTNET_TARGET_FRAMEWORK ([#9454](https://github.com/windmill-labs/windmill/issues/9454)) ([9a609bf](https://github.com/windmill-labs/windmill/commit/9a609bf08ac1b6157dbdfb827fc01e771d71262e))
* sandboxed daemonless container runtime via '# sandbox <image>' ([#9453](https://github.com/windmill-labs/windmill/issues/9453)) ([1727271](https://github.com/windmill-labs/windmill/commit/1727271e197b34026efeaf1b6561bb404a440baa))
* **sandbox:** pull/extract images with crane instead of podman ([#9455](https://github.com/windmill-labs/windmill/issues/9455)) ([7590b28](https://github.com/windmill-labs/windmill/commit/7590b281085afd1fc2774e8fb37a4c0af3aedbad))
### Bug Fixes
* distinguish canceled jobs in runs ([#9452](https://github.com/windmill-labs/windmill/issues/9452)) ([9067787](https://github.com/windmill-labs/windmill/commit/90677872f6185eb0c81e0e84a426a54653818457))
## [1.717.1](https://github.com/windmill-labs/windmill/compare/v1.717.0...v1.717.1) (2026-06-04)
### Bug Fixes
* invalidate relative-import cache when imported script changes ([#9443](https://github.com/windmill-labs/windmill/issues/9443)) ([f595787](https://github.com/windmill-labs/windmill/commit/f595787409a3fcda9278bbcf2cfcc80092f16460))
## [1.717.0](https://github.com/windmill-labs/windmill/compare/v1.716.0...v1.717.0) (2026-06-04)
### Features
* let flow AI chat create and edit sticky notes ([#9412](https://github.com/windmill-labs/windmill/issues/9412)) ([e4e0984](https://github.com/windmill-labs/windmill/commit/e4e0984e55afd3c73f1c365cd0608493a9fd87ed))
### Bug Fixes
* **cli:** push whole raw app instead of treating frontend files as scripts ([#9442](https://github.com/windmill-labs/windmill/issues/9442)) ([b5a6a1e](https://github.com/windmill-labs/windmill/commit/b5a6a1eeab663c2d6aaec2c89eab7a550cb0bb6b))
* read latest db draft for scripts/flows in global mode read tool ([#9441](https://github.com/windmill-labs/windmill/issues/9441)) ([819ba5e](https://github.com/windmill-labs/windmill/commit/819ba5e150ec9f5199919fbea50874fc156d0189))
## [1.716.0](https://github.com/windmill-labs/windmill/compare/v1.715.0...v1.716.0) (2026-06-03)
### Features
* add metadata generation model setting ([#9418](https://github.com/windmill-labs/windmill/issues/9418)) ([cf5fefb](https://github.com/windmill-labs/windmill/commit/cf5fefb521479170b9dc64b884630c4dac789931))
* auto-generate AI session names ([#9399](https://github.com/windmill-labs/windmill/issues/9399)) ([26b7270](https://github.com/windmill-labs/windmill/commit/26b727041830c9b741668a9ab73e2eb90c7cec74))
* support $f/ and $u/ import path aliases for scripts ([#9378](https://github.com/windmill-labs/windmill/issues/9378)) ([220cd35](https://github.com/windmill-labs/windmill/commit/220cd35cf799c42ebf588bc97a6d8e6f4e97c2e3))
* use metadata model for small AI tasks ([#9431](https://github.com/windmill-labs/windmill/issues/9431)) ([79178f6](https://github.com/windmill-labs/windmill/commit/79178f6f5a7c606a2e05677c6efcbdd84c608325))
### Bug Fixes
* **apps:** relock no longer reverts raw app to a stale version ([#9432](https://github.com/windmill-labs/windmill/issues/9432)) ([073857a](https://github.com/windmill-labs/windmill/commit/073857ac0a9ed54bdeac8f373f7c855fe34eb0ac))
* **security:** scope variable and resource value caches by caller identity ([#9427](https://github.com/windmill-labs/windmill/issues/9427)) ([0ba128a](https://github.com/windmill-labs/windmill/commit/0ba128afe797bd016da60563949ac3abbbfe1978))
## [1.715.0](https://github.com/windmill-labs/windmill/compare/v1.714.1...v1.715.0) (2026-06-03)
### Features
* **frontend:** add rebuild dependency map button to workspace settings ([#9424](https://github.com/windmill-labs/windmill/issues/9424)) ([3b2e748](https://github.com/windmill-labs/windmill/commit/3b2e748daf0a8ec4447c30423068df803f3f9ca2))
### Bug Fixes
* **auth:** filter script/flow listings by token scope (GHSA-2ppx-66jv-wpw5) ([#9426](https://github.com/windmill-labs/windmill/issues/9426)) ([7edf3f0](https://github.com/windmill-labs/windmill/commit/7edf3f02122e20fde1e95e0252e7bda641075326))
* **backend:** authorize single-job read endpoints by job/flow visibility ([#9416](https://github.com/windmill-labs/windmill/issues/9416)) ([89a7a37](https://github.com/windmill-labs/windmill/commit/89a7a377764086911db18252f2478f42f0e1e3ea))
* **mcp:** resolve MCP resource token via caller RLS + SSRF-guard url ([#9428](https://github.com/windmill-labs/windmill/issues/9428)) ([8053266](https://github.com/windmill-labs/windmill/commit/8053266f88bd4c94fc86278412df5a0beeed5e77))
* **nsjail:** precompile python stdlib + raise download rlimit_as ([#9429](https://github.com/windmill-labs/windmill/issues/9429)) ([7031744](https://github.com/windmill-labs/windmill/commit/7031744a199f0bf8b8e35043afa959977e5ecdbd))
* omit temperature for gpt-5+ and o-series models on all providers ([#9422](https://github.com/windmill-labs/windmill/issues/9422)) ([11d1ad9](https://github.com/windmill-labs/windmill/commit/11d1ad9a872d2ec2f14cde35708c84a0c7bdc172))
## [1.714.1](https://github.com/windmill-labs/windmill/compare/v1.714.0...v1.714.1) (2026-06-02)
### Bug Fixes
* **backend:** route //native TypeScript previews to native workers (WIN-2007) ([#9407](https://github.com/windmill-labs/windmill/issues/9407)) ([73edebc](https://github.com/windmill-labs/windmill/commit/73edebc833a981488a8ea116f4f13c020a011a6f))
* **nsjail:** raise python download fd limit for --compile-bytecode (WIN-2009) ([#9414](https://github.com/windmill-labs/windmill/issues/9414)) ([9e6559a](https://github.com/windmill-labs/windmill/commit/9e6559a6f688cc8d982277b19920219ea6d0fd8e))
* **triggers:** prevent Zoom challenge handler from being used as a signing oracle ([#9413](https://github.com/windmill-labs/windmill/issues/9413)) ([ab2a15b](https://github.com/windmill-labs/windmill/commit/ab2a15b2a859096eabde718bf6e60289ae187118))
## [1.714.0](https://github.com/windmill-labs/windmill/compare/v1.713.1...v1.714.0) (2026-06-02)
### Features
* add global ai chat test tools ([#9391](https://github.com/windmill-labs/windmill/issues/9391)) ([5c20d6b](https://github.com/windmill-labs/windmill/commit/5c20d6b4f79f2ccc1987ce7fdaf74e6b8f697846))
* add workspace datatable tools to global AI chat mode ([#9395](https://github.com/windmill-labs/windmill/issues/9395)) ([943ef6e](https://github.com/windmill-labs/windmill/commit/943ef6eb2089f4b744cfa7945ce47f7f3b361ec7))
* **flow-ai:** constrain flow-group colors to the NoteColor palette ([#9343](https://github.com/windmill-labs/windmill/issues/9343)) ([e4213c1](https://github.com/windmill-labs/windmill/commit/e4213c1ab8c448f492f372580f5c9df37e33fffc))
* **frontend:** surface local drafts in drawer editors with an unsaved-changes banner ([#9335](https://github.com/windmill-labs/windmill/issues/9335)) ([075faab](https://github.com/windmill-labs/windmill/commit/075faabf3bba16a10a02ae3973008e5a13473085))
* handle CTRL_BREAK_EVENT for graceful shutdown on Windows ([#9400](https://github.com/windmill-labs/windmill/issues/9400)) ([2e14456](https://github.com/windmill-labs/windmill/commit/2e1445616a412c5112ad2247b4087c7ddc218845))
* refine ask-user-question chat display and keyboard nav ([#9392](https://github.com/windmill-labs/windmill/issues/9392)) ([1275487](https://github.com/windmill-labs/windmill/commit/1275487f028d4c74a9eeb18981ed05c225505be0))
* sessions page with isolated AI chat + flow editor ([#9034](https://github.com/windmill-labs/windmill/issues/9034)) ([eadeac2](https://github.com/windmill-labs/windmill/commit/eadeac248bd022c2796cfe638eb617c6143b8fc4))
### Bug Fixes
* **cli:** make encryption key push non-interactive-safe + add --skip-reencrypt-on-key-change ([#9402](https://github.com/windmill-labs/windmill/issues/9402)) ([e356bb1](https://github.com/windmill-labs/windmill/commit/e356bb1f5df92eca3fbb0ca2114b9f4c32d4c496))
* **cli:** stop git-sync promotion deploys from dropping triggers/schedules ([#9403](https://github.com/windmill-labs/windmill/issues/9403)) ([24e3ef2](https://github.com/windmill-labs/windmill/commit/24e3ef27be8498fb820c228a52febf6a0a91b487))
* **frontend:** align Monaco editor font size with text-xs ([#9161](https://github.com/windmill-labs/windmill/issues/9161)) ([de76668](https://github.com/windmill-labs/windmill/commit/de76668c10c04abe8771a8ca7bba7b2259819a1c))
* resolve username rename failing on apps with runnable deps ([#9401](https://github.com/windmill-labs/windmill/issues/9401)) ([e8ad53d](https://github.com/windmill-labs/windmill/commit/e8ad53dae92597f5a1a8b76f38a7d8c24f578a47))
### Performance Improvements
* **python:** add --compile-bytecode to uv pip install ([#9393](https://github.com/windmill-labs/windmill/issues/9393)) ([c19441b](https://github.com/windmill-labs/windmill/commit/c19441bc8cb2da064e4ad44d77dc04ab8bbb22ec))
## [1.713.1](https://github.com/windmill-labs/windmill/compare/v1.713.0...v1.713.1) (2026-06-01)
### Bug Fixes
* **api:** handle multi-version scripts when removing granular ACL ([#9388](https://github.com/windmill-labs/windmill/issues/9388)) ([9d9c503](https://github.com/windmill-labs/windmill/commit/9d9c5038ce8b0016320a670c434ef9063cb40441))
## [1.713.0](https://github.com/windmill-labs/windmill/compare/v1.712.0...v1.713.0) (2026-05-31)
### Features
* **flows:** preserve step/subflow worker tags under a custom-tagged flow ([#9375](https://github.com/windmill-labs/windmill/issues/9375)) ([f0301b1](https://github.com/windmill-labs/windmill/commit/f0301b1605cee5fba4024803555333e6fa5c40ee))
* **oauth:** support per-provider sandbox URLs ([#9358](https://github.com/windmill-labs/windmill/issues/9358)) ([2bf11dc](https://github.com/windmill-labs/windmill/commit/2bf11dcb15540c538ea2ac3cf70dcbe589060b4e))
### Bug Fixes
* **ai:** validate token_url for SSRF in OAuth credentials flow ([#9385](https://github.com/windmill-labs/windmill/issues/9385)) ([4b06881](https://github.com/windmill-labs/windmill/commit/4b06881918b76c5a411cc70b318e46efcc1393a7))
* **api:** authorize and harden log-file reading endpoints ([#9368](https://github.com/windmill-labs/windmill/issues/9368)) ([bb90f4c](https://github.com/windmill-labs/windmill/commit/bb90f4ce83a0e60af219b11c12ab4fe1d13f47a4))
* **apps:** make public apps opt into cross-origin isolation via wm_coep (GIT-884) ([#9374](https://github.com/windmill-labs/windmill/issues/9374)) ([2c0c2c4](https://github.com/windmill-labs/windmill/commit/2c0c2c467f163cd24c14c7be2db07af9cf2ce020))
* **auth:** enforce monotonic privilege on user token lifecycle endpoints ([#9371](https://github.com/windmill-labs/windmill/issues/9371)) ([2ddf93d](https://github.com/windmill-labs/windmill/commit/2ddf93de96622b2a1b2b6f59398a7a1f59360efd))
* batch encryption-key rotation into one git-sync job ([#9355](https://github.com/windmill-labs/windmill/issues/9355)) ([04a0897](https://github.com/windmill-labs/windmill/commit/04a08976aec4ba9b0516350316df303e9f96bfd3))
* **cli:** preserve user drafts on sync push and permissioned-as ([#9381](https://github.com/windmill-labs/windmill/issues/9381)) ([b0c3b01](https://github.com/windmill-labs/windmill/commit/b0c3b01d31b0ab3a6566e1f5fec60e3e230cfadb))
* **frontend:** sanitize user markdown to prevent stored XSS ([#9386](https://github.com/windmill-labs/windmill/issues/9386)) ([def01b8](https://github.com/windmill-labs/windmill/commit/def01b8ff6f331cc36ce02b947adc31c766042c4))
* **security:** re-pin cached hub scripts to CVE-patched versions (+ HUB_BASE_URL override for cache mode) ([#9387](https://github.com/windmill-labs/windmill/issues/9387)) ([edf340c](https://github.com/windmill-labs/windmill/commit/edf340c4d4f18b16b142cb7deb67afa586f10946))
## [1.712.0](https://github.com/windmill-labs/windmill/compare/v1.711.0...v1.712.0) (2026-05-28)
### Features
* add deepseek fim support ([#9365](https://github.com/windmill-labs/windmill/issues/9365)) ([2553fbf](https://github.com/windmill-labs/windmill/commit/2553fbfe31417bd985e7994eac695bf918f97ce2))
* deploy raw apps from global chat ([#9349](https://github.com/windmill-labs/windmill/issues/9349)) ([dec58e6](https://github.com/windmill-labs/windmill/commit/dec58e6c4f55062b42a752c43c89ef05903e713a))
* inject active editor into global chat ([#9361](https://github.com/windmill-labs/windmill/issues/9361)) ([9e7eaf3](https://github.com/windmill-labs/windmill/commit/9e7eaf36847ad3a004ec84e8b7d4784771b7b451))
* **queue:** duration-weighted fairness admission ([#9334](https://github.com/windmill-labs/windmill/issues/9334)) ([045d120](https://github.com/windmill-labs/windmill/commit/045d12043e7c99830ef90bc0da798c94e2094711))
* warn when custom instance db is shared across workspaces ([#9359](https://github.com/windmill-labs/windmill/issues/9359)) ([a9e5140](https://github.com/windmill-labs/windmill/commit/a9e514099585e5ee72df21bd551a223cceb20fb0))
### Bug Fixes
* **cli:** redact encryption_key diff in stdout by default ([#9347](https://github.com/windmill-labs/windmill/issues/9347)) ([88056f8](https://github.com/windmill-labs/windmill/commit/88056f8d4c91c1d14d85a08851ecf0bd97e2260d))
* **cli:** stop re-prompting on wmill refresh prompts ([#9357](https://github.com/windmill-labs/windmill/issues/9357)) ([c2b5ba8](https://github.com/windmill-labs/windmill/commit/c2b5ba8871abbbcff6de69c90e2f09fee70586c1))
* **frontend:** close other sidebar menus when hovering Help ([#9354](https://github.com/windmill-labs/windmill/issues/9354)) ([da882c5](https://github.com/windmill-labs/windmill/commit/da882c54b21e3eaf2c1d1abccd0996b243d96dce))
* **frontend:** prevent duplicate asset node ids crashing flow graph ([#9367](https://github.com/windmill-labs/windmill/issues/9367)) ([9a659b6](https://github.com/windmill-labs/windmill/commit/9a659b636d713ee8fdfbdad41c58bb3d7c79e0d9))
* **frontend:** prevent MultiSelect crash on undefined value ([#9364](https://github.com/windmill-labs/windmill/issues/9364)) ([aea0061](https://github.com/windmill-labs/windmill/commit/aea00611c41379be2afdad0eedd608c9537d03f7))
* **git-sync:** publish fork branch on only_create_branch from the CLI ([#9366](https://github.com/windmill-labs/windmill/issues/9366)) ([2fdc51e](https://github.com/windmill-labs/windmill/commit/2fdc51e62985fc755884436130bdd58e294247c8))
* infer script arg schema when deploying via AI chat ([#9356](https://github.com/windmill-labs/windmill/issues/9356)) ([4efc372](https://github.com/windmill-labs/windmill/commit/4efc37212a98571214aba135b0fbb10dc263fd4f))
* **monitor:** cleanup stale server_heartbeat background_task_state rows ([#9338](https://github.com/windmill-labs/windmill/issues/9338)) ([59ab038](https://github.com/windmill-labs/windmill/commit/59ab038d7718d8a4c25efa5928f42e1393ebbf40))
## [1.711.0](https://github.com/windmill-labs/windmill/compare/v1.710.1...v1.711.0) (2026-05-26)
### Features
* **cli:** add object-storage commands and flow test-step ([#9326](https://github.com/windmill-labs/windmill/issues/9326)) ([36f574f](https://github.com/windmill-labs/windmill/commit/36f574ff951198a4d40ee068a27d74c41ce32154))
### Bug Fixes
* **cli:** handle __flow suffix when deriving the flow's Windmill path ([#9333](https://github.com/windmill-labs/windmill/issues/9333)) ([6f77034](https://github.com/windmill-labs/windmill/commit/6f770346fb330997a836c39fba347df4c088a83c))
* **queue:** duration-weighted workspace fairness signal ([#9329](https://github.com/windmill-labs/windmill/issues/9329)) ([42d2121](https://github.com/windmill-labs/windmill/commit/42d2121af925de50f549ecb72ffb5132f5c41079))
## [1.710.1](https://github.com/windmill-labs/windmill/compare/v1.710.0...v1.710.1) (2026-05-26)
### Bug Fixes
* improve workspace fairness ([896add0](https://github.com/windmill-labs/windmill/commit/896add0350f4de31f5674d6be0907a582c5ec17e))
## [1.710.0](https://github.com/windmill-labs/windmill/compare/v1.709.0...v1.710.0) (2026-05-26)
### Features
* **queue:** stochastic admission + EE availability of workspace fairness algorithm ([#9321](https://github.com/windmill-labs/windmill/issues/9321)) ([8bf7fd2](https://github.com/windmill-labs/windmill/commit/8bf7fd2c921c48861b71731a085b18ea8f72fb68))
### Bug Fixes
* **websocket-trigger:** honor HTTPS_PROXY/HTTP_PROXY/NO_PROXY ([#9324](https://github.com/windmill-labs/windmill/issues/9324)) ([6f36316](https://github.com/windmill-labs/windmill/commit/6f363163df9cd15f5af7d56cf34a01b70d236830))
## [1.709.0](https://github.com/windmill-labs/windmill/compare/v1.708.0...v1.709.0) (2026-05-25)
### Features
* add copy button to Path component ([#9311](https://github.com/windmill-labs/windmill/issues/9311)) ([98bd5e7](https://github.com/windmill-labs/windmill/commit/98bd5e7f2a437b8b534028838b6ed0d7c59f7011))
* **ai-chat:** align footer bar + DropdownV2 mode/autonomy selectors ([#9308](https://github.com/windmill-labs/windmill/issues/9308)) ([2f50e8b](https://github.com/windmill-labs/windmill/commit/2f50e8bab0b5ae9ae297c79abfe96df441f405e2))
* **ai-chat:** expand chat question answers ([#9310](https://github.com/windmill-labs/windmill/issues/9310)) ([3f219ae](https://github.com/windmill-labs/windmill/commit/3f219aed98d93158aefce01bb51ed12dcb4711a1))
* plug global chat drafts into userdraft ([#9291](https://github.com/windmill-labs/windmill/issues/9291)) ([1eef531](https://github.com/windmill-labs/windmill/commit/1eef53170b1b2afb75b9812e33787d1f28cf50dd))
* **raw_apps:** surface UI Builder build errors over the preview pane ([#9316](https://github.com/windmill-labs/windmill/issues/9316)) ([90a196d](https://github.com/windmill-labs/windmill/commit/90a196d8d81993ffc2377d7088ab98f7b0f5ddcc))
* **raw_apps:** tab-based editor surface with split-with-preview ([#9273](https://github.com/windmill-labs/windmill/issues/9273)) ([368e677](https://github.com/windmill-labs/windmill/commit/368e6774194a58058f28d1b4a42f8f4a7ec4ab63))
* **service-accounts:** allow choosing role at creation time ([#9307](https://github.com/windmill-labs/windmill/issues/9307)) ([b125eca](https://github.com/windmill-labs/windmill/commit/b125eca7628b07c071bd102b161d389259fd6c62))
### Bug Fixes
* **auth:** filter resource/variable listings by token scope (WIN-1981) ([#9302](https://github.com/windmill-labs/windmill/issues/9302)) ([b5a0d46](https://github.com/windmill-labs/windmill/commit/b5a0d46695fdfe692d64573d1cfa06511e3b33f5))
* **jobs:** authorization bypass in only_result job updates (WIN-1980) ([#9301](https://github.com/windmill-labs/windmill/issues/9301)) ([108a88a](https://github.com/windmill-labs/windmill/commit/108a88a1801548c8570d56aa3e1eb80246367bf4))
## [1.708.0](https://github.com/windmill-labs/windmill/compare/v1.707.0...v1.708.0) (2026-05-24)
Dockerfile
+19 -5
View File
@@ -66,6 +66,7 @@ RUN npm ci
COPY frontend .
RUN mkdir /backend
COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml
COPY /backend/oauth_connect.json /backend/oauth_connect.json
COPY /openflow.openapi.yaml /openflow.openapi.yaml
COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh
COPY /system_prompts/auto-generated /system_prompts/auto-generated
@@ -232,11 +233,14 @@ ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.9.25/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
# Preinstall python runtimes to temp build location (will copy with world-writable perms later)
RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11
RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY
# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run
# under the read-only nsjail runtime mount (uv >= 0.9.25). The copy below MUST preserve
# timestamps or Python's mtime-based .pyc invalidation discards these compiled files.
RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install 3.11 --compile-bytecode
RUN UV_CACHE_DIR=/tmp/build_cache/uv UV_PYTHON_INSTALL_DIR=/tmp/build_cache/py_runtime uv python install $LATEST_STABLE_PY --compile-bytecode
RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
@@ -258,7 +262,7 @@ RUN export GOCACHE=/tmp/build_cache/go && \
# chmod a+rw adds read+write WITHOUT removing execute bits (755->777, 644->666)
# Note: uv python install only creates py_runtime, not uv cache - we create uv/go dirs for runtime
RUN mkdir -p /tmp/windmill/cache && \
cp -r /tmp/build_cache/* /tmp/windmill/cache/ && \
cp -r --preserve=timestamps /tmp/build_cache/* /tmp/windmill/cache/ && \
chmod -R a+rw /tmp/windmill/cache && \
rm -rf /tmp/build_cache && \
mkdir -p -m 777 /tmp/windmill/cache/uv /tmp/windmill/cache/go /tmp/windmill/cache/rustup /tmp/windmill/cache/cargo
@@ -299,10 +303,20 @@ ENV CARGO_HOME="/tmp/windmill/cache/cargo"
ENV LD_LIBRARY_PATH="."
# nsjail runtime deps and binary
RUN apt-get update && apt-get install -y libprotobuf-dev libnl-route-3-dev \
RUN apt-get update && apt-get install -y --no-install-recommends libprotobuf32 libnl-route-3-200 libnl-3-200 \
&& apt-get clean && rm -rf /var/lib/apt/lists/*
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
# crane: pulls + flattens images for the sandboxed container runtime (`# sandbox <image>`).
# Single static binary — no daemon/store/root needed. See docs/docker-v2-runtime.md.
ARG CRANE_VERSION=v0.20.6
RUN arch="$(dpkg --print-architecture)"; \
case "$arch" in amd64) crane_arch=x86_64 ;; arm64) crane_arch=arm64 ;; *) echo >&2 "error: unsupported arch '$arch' for crane"; exit 1 ;; esac; \
wget -O /tmp/crane.tgz "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \
&& tar -xzf /tmp/crane.tgz -C /usr/local/bin crane \
&& rm /tmp/crane.tgz \
&& chmod +x /usr/local/bin/crane
WORKDIR ${APP}
RUN ln -s ${APP}/windmill /usr/local/bin/windmill
ai_evals/AGENTS.md
+25
View File
@@ -86,6 +86,31 @@ Global prompts should exercise workspace-level drafting behavior:
Keep deterministic validation focused on the draft contract: required draft type/path, required content snippets, forbidden draft paths, and forbidden mutating tools such as deploy/delete unless the case explicitly asks for them.
Datatable cases should set `skipJudge: true` and validate through tool-use
(`requiredToolsUsed` / `forbiddenToolsUsed`) and SQL-argument assertions
(`toolCallArgs` with `stringIncludesAnyOf`, e.g. `['select']`, `['create table']`,
`['update', 'insert into']`). Two reasons the judge is unreliable here:
- `list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql`
produce no drafts, and the global judge only sees the drafts artifact — it
scores a no-draft conversational answer as empty (same as the
`askUserQuestion` cases).
- Even a case that *does* produce a draft (a script reading the data table via
`wmill.datatable()` at runtime) is mis-judged: the judge has no datatable SDK
reference and penalizes correct `wmill.datatable()` usage as wrong. Verify the
SDK call deterministically instead — `requiredDrafts.valueIncludes: ['wmill.datatable(']`
plus forbidding `exec_datatable_sql` (keeping chat-time SQL distinct from
runtime SDK use).
`stringIncludesAnyOf` is existential over calls (at least one matching call), so a
mutation case still passes when the model mixes its UPDATE/INSERT with
verification SELECTs. The in-memory engine (`datatableSqlEngine.ts`) is stateful
within a case — writes persist, so a model that re-queries to verify its
CREATE/UPDATE sees the change and does not loop. But the engine is best-effort
(SELECT returns all rows of the referenced/first table with no WHERE/projection),
so still never assert specific returned row values. Seed data via
`workspace.datatables` in the `initial` fixture (see README).
## Deterministic validation
Use deterministic validation only for hard failures such as:
ai_evals/README.md
+39 -8
View File
@@ -56,7 +56,7 @@ bun run cli -- run flow flow-test4-order-processing-loop --model opus
bun run cli -- run flow flow-test0-sum-two-numbers --models haiku,opus,4o
bun run cli -- run flow flow-test0-sum-two-numbers --runs 3 --verbose
bun run cli -- run flow --record
GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-pro
GEMINI_API_KEY=... bun run cli -- run app app-test1-counter-create --model gemini-3-flash-preview
WMILL_AI_EVAL_BACKEND_URL=http://127.0.0.1:8000 bun run cli -- run flow --backend-validation preview
bun run cli -- run global global-test1-script-create
bun run cli -- run cli bun-hello-script
@@ -88,15 +88,16 @@ Today:
- `sonnet`
- `opus`
- `4o`
- `gemini-flash`
- `gemini-pro`
- `gpt-5.5`
- `gemini-3-flash-preview`
- `gemini-3.1-pro-preview`
- `deepseek-v4-flash`
- `deepseek-v4-pro`
Notes:
- the command also prints accepted alias spellings such as `gpt-4o`, `claude-opus-4.6`, and `claude-haiku-4.5`
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, and Gemini-backed aliases
- the command also prints accepted alias spellings such as `gpt-4o`, `gpt-55`, `claude-opus-4.6`, and `claude-haiku-4.5`
- frontend modes (`flow`, `script`, `app`, `global`) can use Anthropic, OpenAI, Gemini, and DeepSeek-backed aliases
- `cli` mode always uses the Anthropic agent SDK, so only Anthropic aliases are valid there
- the judge model is separate and currently defaults to `claude-sonnet-4-6`
@@ -142,6 +143,32 @@ For `global` mode, `validate` can express draft-level requirements such as:
- required or forbidden draft counts
- forbidden draft paths
Global initial fixtures can also seed `liveEditorDrafts` with `type`,
`storagePath`, `effectivePath`, and `value` fields. These drafts emulate the
currently open script, flow, or raw app editor so cases can test prompts that
refer to "this" or the "current" item.
Global (and flow) initial fixtures can seed `workspace.datatables` so the
`list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools
return seeded data during evals. Each entry is
`{ datatable_name, schemas: { <schema>: { <table>: { columns, rows? } } } }`.
SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real
database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/
`DELETE` mutate the seeded datatable in place, so a later `list_datatables`,
`get_datatable_table_schema`, `SELECT`, or `information_schema` query reflects them
— this is what stops a model from looping when it re-queries to verify a write.
The engine is best-effort: `SELECT` returns all rows of the referenced (or first)
table with no WHERE filtering/projection/joins, `WHERE` on UPDATE/DELETE supports
`col = value` predicates joined by `AND`, and anything unparseable is a no-op
success. So validate datatable cases through tool-use and SQL-argument assertions
(`requiredToolsUsed`, `stringIncludesAnyOf`) — not through exact returned row
values. An empty/absent `datatables` seed makes `list_datatables` return `[]`,
which is what the "no datatable configured" blocking cases rely on.
Set `WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT=1` to run those cases with
the old behavior where the live editor is only discoverable through
`list_workspace_items`.
App fixtures can also include an optional `datatables.json` file at the fixture root.
For `flow` mode, an `initial` fixture can also include a benchmark workspace catalog of
@@ -189,11 +216,15 @@ If `--record` is used, the CLI also appends one compact JSON line to:
Each recorded line contains:
- run metadata (`createdAt`, `gitSha`, `mode`, `runModel`, `judgeModel`)
- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averageJudgeScore`)
- average token usage (`averageTokenUsagePerAttempt`)
- per-case metrics under `cases[]` (`averageDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, pass rate)
- suite totals (`caseCount`, `attemptCount`, `passedAttempts`, `passRate`, `averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`)
- average token usage (`averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`)
- per-case metrics under `cases[]` (`averageDurationMs`, `averagePassedDurationMs`, `averageJudgeScore`, `averageTokenUsagePerAttempt`, `averageTokenUsagePerPassedAttempt`, pass rate)
- `failedCaseIds`
The CLI headline duration and token averages use passed attempts only.
All-attempt averages are still recorded to make failures auditable without
letting failed attempts skew success cost comparisons.
Example:
- summary: `ai_evals/results/2026-04-09T09-40-33.051Z__flow.json`
ai_evals/adapters/frontend/benchmarkRunner.ts
+15 -11
View File
@@ -10,10 +10,6 @@ import { runSuite } from "../../core/runSuite";
import type { BenchmarkRunResult, ModeRunner } from "../../core/types";
import { resolveWindmillBackendSettings } from "../../core/windmillBackendSettings";
import { emitFrontendBenchmarkProgress } from "./progress";
import { createAppModeRunner } from "../../modes/app";
import { createFlowModeRunner } from "../../modes/flow";
import { createGlobalModeRunner } from "../../modes/global";
import { createScriptModeRunner } from "../../modes/script";
import { DEFAULT_JUDGE_MODEL } from "../../core/judge";
export type FrontendBenchmarkMode = "flow" | "app" | "script" | "global";
@@ -40,7 +36,7 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
const backendSettings = resolveWindmillBackendSettings();
const selectedCases = await loadSelectedCases(mode, caseIds);
const modeRunner = getModeRunner(
const modeRunner = await getModeRunner(
mode,
getFrontendEvalModel(model),
backendValidation,
@@ -69,25 +65,33 @@ export async function runFrontendBenchmarkFromEnv(): Promise<BenchmarkRunResult>
});
}
function getModeRunner(
async function getModeRunner(
mode: FrontendBenchmarkMode,
model: ReturnType<typeof getFrontendEvalModel>,
backendValidation: ReturnType<typeof resolveBackendValidationSettings>,
backendSettings: ReturnType<typeof resolveWindmillBackendSettings>,
): ModeRunner<any, any, any> {
): Promise<ModeRunner<any, any, any>> {
switch (mode) {
case "flow":
case "flow": {
const { createFlowModeRunner } = await import("../../modes/flow");
return createFlowModeRunner(model, backendValidation, backendSettings);
case "app":
}
case "app": {
const { createAppModeRunner } = await import("../../modes/app");
return createAppModeRunner(model, backendSettings);
case "script":
}
case "script": {
const { createScriptModeRunner } = await import("../../modes/script");
return createScriptModeRunner(
model,
backendValidation,
backendSettings,
);
case "global":
}
case "global": {
const { createGlobalModeRunner } = await import("../../modes/global");
return createGlobalModeRunner(model, backendSettings);
}
}
}
ai_evals/adapters/frontend/core/global/globalEvalRunner.ts
+63 -5
View File
@@ -7,8 +7,12 @@ import {
prepareGlobalSystemMessage,
prepareGlobalUserMessage,
} from "../../../../../frontend/src/lib/components/copilot/chat/global/core";
import { globalDraftStore } from "../../../../../frontend/src/lib/components/copilot/chat/global/draftStore.svelte";
import {
clearGlobalDrafts,
listGlobalDrafts,
} from "../../../../../frontend/src/lib/components/copilot/chat/global/userDraftAdapter";
import type { Tool as ProductionTool } from "../../../../../frontend/src/lib/components/copilot/chat/shared";
import { UserDraft } from "../../../../../frontend/src/lib/userDraft.svelte";
import type { ModeRunContext } from "../../../../core/types";
import type { GlobalDraftState } from "../../../../core/validators";
import type { WindmillBackendSettings } from "../../../../core/windmillBackendSettings";
@@ -24,6 +28,21 @@ const MUTATING_GLOBAL_TOOLS = new Set([
"deploy_workspace_item",
"delete_workspace_item",
]);
const DISABLE_ACTIVE_EDITOR_CONTEXT_ENV =
"WMILL_AI_EVAL_DISABLE_ACTIVE_EDITOR_CONTEXT";
const LIVE_EDITOR_ITEM_KINDS = {
script: "script",
flow: "flow",
app: "raw_app",
} as const;
export interface GlobalLiveEditorDraftFixture {
type: keyof typeof LIVE_EDITOR_ITEM_KINDS;
storagePath?: string;
effectivePath?: string;
value?: unknown;
}
export interface GlobalEvalResult {
success: boolean;
@@ -38,6 +57,7 @@ export interface GlobalEvalResult {
export interface GlobalEvalOptions {
workspaceFixtures?: BenchmarkWorkspaceRunnables;
liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
model?: string;
maxIterations?: number;
provider?: AIProvider;
@@ -55,19 +75,26 @@ export async function runGlobalEval(
options.workspaceRoot ??
(await mkdtemp(join(tmpdir(), "wmill-frontend-global-benchmark-")));
globalDraftStore.clearDrafts(workspaceRoot);
clearGlobalDrafts(workspaceRoot);
registerBenchmarkWorkspaceRunnables(workspaceRoot, options.workspaceFixtures ?? {});
seedLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
try {
const model = options.model ?? "claude-haiku-4-5-20251001";
const injectActiveEditorContext =
process.env[DISABLE_ACTIVE_EDITOR_CONTEXT_ENV] !== "1";
const rawResult = await runEval({
userPrompt,
systemMessage: prepareGlobalSystemMessage(),
userMessage: prepareGlobalUserMessage(userPrompt),
userMessage: prepareGlobalUserMessage(
userPrompt,
[],
injectActiveEditorContext ? { workspace: workspaceRoot } : {},
),
tools: getGlobalEvalTools(),
helpers: {},
apiKey,
getOutput: () => ({ drafts: globalDraftStore.listDrafts(workspaceRoot) }),
getOutput: () => ({ drafts: listGlobalDrafts(workspaceRoot) }),
onAssistantMessageStart: options.runContext?.onAssistantMessageStart,
onAssistantToken: options.runContext?.onAssistantChunk,
onAssistantMessageEnd: options.runContext?.onAssistantMessageEnd,
@@ -94,7 +121,8 @@ export async function runGlobalEval(
tokenUsage: rawResult.tokenUsage,
};
} finally {
globalDraftStore.clearDrafts(workspaceRoot);
clearGlobalDrafts(workspaceRoot);
clearLiveEditorDrafts(workspaceRoot, options.liveEditorDrafts ?? []);
unregisterBenchmarkWorkspaceRunnables(workspaceRoot);
if (!options.workspaceRoot) {
await rm(workspaceRoot, { recursive: true, force: true });
@@ -102,6 +130,36 @@ export async function runGlobalEval(
}
}
function seedLiveEditorDrafts(
workspace: string,
fixtures: GlobalLiveEditorDraftFixture[],
): void {
for (const fixture of fixtures) {
const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type];
const storagePath = fixture.storagePath ?? fixture.effectivePath ?? "";
if (fixture.value !== undefined) {
UserDraft.save(itemKind, storagePath, fixture.value, { workspace });
}
UserDraft.setLiveEditorDraft({
workspace,
itemKind,
storagePath,
effectivePath: fixture.effectivePath ?? fixture.storagePath,
});
}
}
function clearLiveEditorDrafts(
workspace: string,
fixtures: GlobalLiveEditorDraftFixture[],
): void {
for (const fixture of fixtures) {
const itemKind = LIVE_EDITOR_ITEM_KINDS[fixture.type];
const storagePath = fixture.storagePath ?? fixture.effectivePath ?? "";
UserDraft.clearLiveEditorDraft(itemKind, { workspace, storagePath });
}
}
function getGlobalEvalTools(): ProductionTool<{}>[] {
return (globalTools as ProductionTool<{}>[]).map((tool) => {
if (!MUTATING_GLOBAL_TOOLS.has(tool.def.function.name)) {
ai_evals/adapters/frontend/core/shared/providerConfig.test.ts
+6 -4
View File
@@ -21,9 +21,9 @@ describe("proxy helpers", () => {
describe("resolveEvalModelProvider", () => {
it("infers googleai from Gemini model ids", () => {
expect(resolveEvalModelProvider("gemini-2.5-flash")).toEqual({
expect(resolveEvalModelProvider("gemini-3-flash-preview")).toEqual({
provider: "googleai",
model: "gemini-2.5-flash",
model: "gemini-3-flash-preview",
});
});
@@ -35,9 +35,11 @@ describe("resolveEvalModelProvider", () => {
});
it("preserves an explicit provider", () => {
expect(resolveEvalModelProvider("gemini-2.5-pro", "googleai")).toEqual({
expect(
resolveEvalModelProvider("gemini-3.1-pro-preview", "googleai"),
).toEqual({
provider: "googleai",
model: "gemini-2.5-pro",
model: "gemini-3.1-pro-preview",
});
});
});
ai_evals/adapters/frontend/datatableSqlEngine.test.ts
+262
View File
@@ -0,0 +1,262 @@
import { describe, expect, it } from 'bun:test'
import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine'
function makeDatatable(): BenchmarkDatatableSeed {
return {
datatable_name: 'main',
schemas: {
public: {
orders: {
columns: { id: 'int4', customer_id: 'int4', total: 'numeric', status: 'text' },
rows: [
{ id: 1, customer_id: 1, total: 42.5, status: 'shipped' },
{ id: 2, customer_id: 2, total: 19.99, status: 'pending' },
{ id: 3, customer_id: 1, total: 88, status: 'shipped' }
]
},
customers: {
columns: { id: 'int4', name: 'text' },
rows: [{ id: 1, name: 'Alice' }]
}
}
}
}
}
describe('SELECT', () => {
it('returns the referenced table rows', () => {
const dt = makeDatatable()
expect(applyDatatableSql(dt, 'SELECT id, name FROM customers').rows).toEqual([
{ id: 1, name: 'Alice' }
])
})
it('falls back to the first table when no known table is referenced', () => {
const dt = makeDatatable()
expect(applyDatatableSql(dt, 'select 1').rows).toHaveLength(3)
})
it('resolves a schema-qualified table', () => {
const dt = makeDatatable()
expect(applyDatatableSql(dt, 'SELECT * FROM public.customers').rows).toEqual([
{ id: 1, name: 'Alice' }
])
})
})
describe('CREATE TABLE', () => {
it('adds a table with parsed columns, skipping table constraints and FK clauses', () => {
const dt = makeDatatable()
const result = applyDatatableSql(
dt,
'CREATE TABLE public.refunds (\n order_id int4 NOT NULL REFERENCES public.orders(id),\n amount numeric(10,2),\n PRIMARY KEY (order_id)\n)'
)
expect(result.rows).toEqual([])
expect(dt.schemas.public.refunds).toEqual({
columns: { order_id: 'int4', amount: 'numeric(10,2)' },
rows: []
})
})
it('defaults an unqualified table to the public schema', () => {
const dt = makeDatatable()
applyDatatableSql(dt, 'CREATE TABLE notes (id int4, body text)')
expect(dt.schemas.public.notes.columns).toEqual({ id: 'int4', body: 'text' })
})
it('is a no-op for an existing table with IF NOT EXISTS', () => {
const dt = makeDatatable()
applyDatatableSql(dt, 'CREATE TABLE IF NOT EXISTS public.orders (x int4)')
expect(Object.keys(dt.schemas.public.orders.columns)).toContain('status')
})
})
describe('DROP TABLE', () => {
it('removes the table', () => {
const dt = makeDatatable()
applyDatatableSql(dt, 'DROP TABLE IF EXISTS public.customers')
expect(dt.schemas.public.customers).toBeUndefined()
})
})
describe('INSERT', () => {
it('appends a row using an explicit column list', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (2, 'Bob')")
expect(dt.schemas.public.customers.rows).toContainEqual({ id: 2, name: 'Bob' })
})
it('infers columns from the table when none are given, and appends multiple tuples', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "INSERT INTO customers VALUES (2, 'Bob'), (3, 'Carol')")
expect(dt.schemas.public.customers.rows).toHaveLength(3)
})
it('returns the inserted rows when RETURNING is present', () => {
const dt = makeDatatable()
const result = applyDatatableSql(
dt,
"INSERT INTO customers (id, name) VALUES (2, 'Bob') RETURNING *"
)
expect(result.rows).toEqual([{ id: 2, name: 'Bob' }])
})
})
describe('UPDATE', () => {
it('updates only the rows matching an equality WHERE', () => {
const dt = makeDatatable()
const result = applyDatatableSql(
dt,
"UPDATE public.orders SET status = 'shipped' WHERE id = 2"
)
expect(result.rows).toEqual([])
expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('shipped')
expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped')
})
it('strips a Postgres cast in the WHERE value', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "UPDATE orders SET status = 'done' WHERE id = 2::int4")
expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done')
})
it('matches multiple AND predicates including a numeric literal', () => {
const dt = makeDatatable()
applyDatatableSql(
dt,
"UPDATE orders SET status = 'done' WHERE customer_id = 2 AND total = 19.99"
)
expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('done')
expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped')
})
it('updates every row when there is no WHERE', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "UPDATE orders SET status = 'archived'")
expect(dt.schemas.public.orders.rows?.every((r) => r.status === 'archived')).toBe(true)
})
it('returns the affected rows when RETURNING is present', () => {
const dt = makeDatatable()
const result = applyDatatableSql(
dt,
"UPDATE orders SET status = 'shipped' WHERE id = 2 RETURNING *"
)
expect(result.rows).toHaveLength(1)
expect(result.rows[0]).toMatchObject({ id: 2, status: 'shipped' })
})
it('affects no rows when the WHERE clause cannot be parsed', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "UPDATE orders SET status = 'x' WHERE total > 20")
expect(dt.schemas.public.orders.rows?.some((r) => r.status === 'x')).toBe(false)
})
})
describe('DELETE', () => {
it('removes only the matching rows', () => {
const dt = makeDatatable()
applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2')
expect(dt.schemas.public.orders.rows?.map((r) => r.id)).toEqual([1, 3])
})
it('returns the removed rows when RETURNING is present', () => {
const dt = makeDatatable()
const result = applyDatatableSql(dt, 'DELETE FROM orders WHERE id = 2 RETURNING *')
expect(result.rows).toEqual([{ id: 2, customer_id: 2, total: 19.99, status: 'pending' }])
})
})
describe('writes are reflected by later reads', () => {
it('UPDATE then SELECT sees the new value (the verify-loop fix)', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "UPDATE orders SET status = 'shipped' WHERE id = 2")
const seen = applyDatatableSql(dt, 'SELECT * FROM orders').rows
expect(seen.find((r) => r.id === 2)?.status).toBe('shipped')
})
it('INSERT then SELECT sees the new row', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (9, 'Zed')")
const seen = applyDatatableSql(dt, 'SELECT * FROM customers').rows
expect(seen).toContainEqual({ id: 9, name: 'Zed' })
})
it('CREATE then SELECT on the new table returns its (empty) rows', () => {
const dt = makeDatatable()
applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4, amount numeric)')
expect(applyDatatableSql(dt, 'SELECT * FROM refunds').rows).toEqual([])
})
})
describe('system-catalog queries reflect the current tables/columns', () => {
it('lists current tables (including a freshly created one) via information_schema.tables', () => {
const dt = makeDatatable()
applyDatatableSql(dt, 'CREATE TABLE public.refunds (order_id int4)')
const rows = applyDatatableSql(
dt,
"SELECT table_name FROM information_schema.tables WHERE table_name = 'refunds'"
).rows
expect(rows.map((r) => r.table_name)).toContain('refunds')
})
it('does not list a dropped table', () => {
const dt = makeDatatable()
applyDatatableSql(dt, 'DROP TABLE public.customers')
const rows = applyDatatableSql(dt, 'SELECT table_name FROM information_schema.tables').rows
expect(rows.map((r) => r.table_name)).not.toContain('customers')
})
it('reports columns via information_schema.columns', () => {
const dt = makeDatatable()
const rows = applyDatatableSql(
dt,
"SELECT column_name FROM information_schema.columns WHERE table_name = 'orders'"
).rows
expect(rows.map((r) => r.column_name)).toContain('status')
})
})
describe('parser robustness (string/paren-aware splitting)', () => {
it('does not treat the word "returning" inside a string value as a RETURNING clause', () => {
const dt = makeDatatable()
const result = applyDatatableSql(
dt,
"INSERT INTO customers (id, name) VALUES (5, 'is returning soon')"
)
expect(result.rows).toEqual([])
expect(dt.schemas.public.customers.rows).toContainEqual({ id: 5, name: 'is returning soon' })
})
it('does not split on the word "where" inside a SET string value', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "UPDATE orders SET status = 'ship where ordered' WHERE id = 2")
expect(dt.schemas.public.orders.rows?.find((r) => r.id === 2)?.status).toBe('ship where ordered')
expect(dt.schemas.public.orders.rows?.find((r) => r.id === 1)?.status).toBe('shipped')
})
it('keeps INSERT tuples intact when a value contains a function call', () => {
const dt = makeDatatable()
applyDatatableSql(dt, "INSERT INTO customers (id, name) VALUES (6, coalesce(NULL, 'x'))")
expect(dt.schemas.public.customers.rows).toHaveLength(2)
expect(dt.schemas.public.customers.rows?.[1]).toMatchObject({ id: 6 })
})
it('CREATE TABLE ignores a trailing semicolon-separated statement', () => {
const dt = makeDatatable()
applyDatatableSql(
dt,
'CREATE TABLE public.refunds (id int4, amount numeric); INSERT INTO refunds VALUES (1, 5)'
)
expect(dt.schemas.public.refunds.columns).toEqual({ id: 'int4', amount: 'numeric' })
expect(dt.schemas.public.refunds.rows).toEqual([])
})
})
describe('unparseable statements are a safe no-op', () => {
it('returns [] and does not throw', () => {
const dt = makeDatatable()
expect(applyDatatableSql(dt, 'VACUUM ANALYZE').rows).toEqual([])
expect(applyDatatableSql(dt, 'GRANT SELECT ON orders TO someone').rows).toEqual([])
})
})
ai_evals/adapters/frontend/datatableSqlEngine.ts
+541
View File
@@ -0,0 +1,541 @@
/**
* A deliberately small, best-effort SQL engine for the benchmark datatable mock.
*
* This is NOT a real SQL implementation — it exists only so that writes a model
* issues during an eval (`CREATE TABLE`, `INSERT`, `UPDATE`, `DELETE`, `DROP`)
* become visible to its later reads (`list_datatables`, `get_datatable_table_schema`,
* `SELECT`). Without that, a model that re-queries to verify a write sees stale
* seed data, concludes the write failed, and loops until it exhausts its turns.
*
* It parses only the common statement shapes models produce. Anything it cannot
* parse is a no-op success (it never throws) — behavioral evals assert that the
* right statement was issued, not its exact data effects. Notable limits:
* - `SELECT` returns all rows of the referenced (or first) table — no WHERE
* filtering, projection, joins, or aggregation.
* - `WHERE` supports `col = value` predicates joined by `AND` only; an
* unparseable WHERE on UPDATE/DELETE affects zero rows (never the whole table).
*/
/** One seeded datatable table: its columns (col -> compact_type) and optional rows. */
export interface BenchmarkDatatableTableSeed {
columns: Record<string, string>
rows?: Record<string, unknown>[]
}
/** A seeded datatable: `datatable_name` plus a `schema -> table -> seed` map. */
export interface BenchmarkDatatableSeed {
datatable_name: string
schemas: {
[schema: string]: {
[table: string]: BenchmarkDatatableTableSeed
}
}
}
export interface DatatableSqlResult {
rows: Record<string, unknown>[]
}
const DEFAULT_SCHEMA = 'public'
type ParsedRef = { schema: string; table: string }
type Predicate = { column: string; value: unknown }
/**
* Apply one SQL statement to `datatable` IN PLACE and return the result rows.
* SELECT returns the referenced/first table's rows; a mutation returns its
* affected rows when it has a RETURNING clause, otherwise `[]`.
*/
export function applyDatatableSql(
datatable: BenchmarkDatatableSeed,
sql: string
): DatatableSqlResult {
const statement = stripTrailingSemicolon(sql.trim())
if (/^\s*(with|select)\b/i.test(statement)) {
return { rows: selectRows(datatable, statement) }
}
if (/^\s*create\s+table\b/i.test(statement)) {
return { rows: applyCreateTable(datatable, statement) }
}
if (/^\s*drop\s+table\b/i.test(statement)) {
return { rows: applyDropTable(datatable, statement) }
}
if (/^\s*insert\s+into\b/i.test(statement)) {
return { rows: applyInsert(datatable, statement) }
}
if (/^\s*update\b/i.test(statement)) {
return { rows: applyUpdate(datatable, statement) }
}
if (/^\s*delete\s+from\b/i.test(statement)) {
return { rows: applyDelete(datatable, statement) }
}
return { rows: [] }
}
// ============= Reads =============
function selectRows(
datatable: BenchmarkDatatableSeed,
sql: string
): Record<string, unknown>[] {
const fromRef = sql.match(/\bfrom\s+([a-zA-Z_"][\w."]*)/i)?.[1]
if (fromRef) {
const catalog = catalogRows(datatable, fromRef)
if (catalog) {
return catalog
}
}
const table = fromRef ? resolveTable(datatable, fromRef) : undefined
const seed = table ?? firstTable(datatable)
return seed?.rows ?? []
}
/**
* Synthesize rows for a system-catalog query so a model verifying a `CREATE`/`DROP`
* via `information_schema.tables` / `.columns` (or `pg_tables`) sees the current
* tables/columns instead of fallback data. WHERE is not applied, so the model gets
* the full set and finds (or no longer finds) the table it just changed.
* Returns `undefined` for non-catalog refs so normal table resolution proceeds.
*/
function catalogRows(
datatable: BenchmarkDatatableSeed,
ref: string
): Record<string, unknown>[] | undefined {
const normalized = ref.toLowerCase().replace(/"/g, '')
const name = normalized.split('.').pop()
const isCatalog = normalized.includes('information_schema.') || normalized.startsWith('pg_')
if (!isCatalog) {
return undefined
}
const tables = allTables(datatable)
if (name === 'tables' || name === 'pg_tables') {
return tables.map(({ schema, table }) => ({
table_schema: schema,
table_name: table,
schemaname: schema,
tablename: table
}))
}
if (name === 'columns') {
return tables.flatMap(({ schema, table, seed }) =>
Object.entries(seed.columns).map(([column, type]) => ({
table_schema: schema,
table_name: table,
column_name: column,
data_type: type
}))
)
}
return undefined
}
function allTables(
datatable: BenchmarkDatatableSeed
): { schema: string; table: string; seed: BenchmarkDatatableTableSeed }[] {
return Object.entries(datatable.schemas).flatMap(([schema, tables]) =>
Object.entries(tables).map(([table, seed]) => ({ schema, table, seed }))
)
}
// ============= DDL =============
function applyCreateTable(
datatable: BenchmarkDatatableSeed,
sql: string
): Record<string, unknown>[] {
const head = sql.match(
/^\s*create\s+table\s+(?:if\s+not\s+exists\s+)?([a-zA-Z_"][\w."]*)/i
)
// The first top-level paren group is the column-definition list; using it (rather
// than a greedy `(...)` capture) ignores any trailing `;`-separated statement.
const columnText = extractParenGroups(sql)[0]
if (!head || columnText === undefined) {
return []
}
const { schema, table } = parseRef(head[1])
const existing = datatable.schemas[schema]?.[table]
if (existing) {
return []
}
const columns: Record<string, string> = {}
for (const rawDef of splitTopLevel(columnText)) {
const def = rawDef.trim()
if (!def || isTableConstraint(def)) {
continue
}
const tokens = def.split(/\s+/)
const column = unquoteIdentifier(tokens[0])
if (!column) {
continue
}
columns[column] = tokens[1] ?? 'text'
}
if (!datatable.schemas[schema]) {
datatable.schemas[schema] = {}
}
datatable.schemas[schema][table] = { columns, rows: [] }
return []
}
function applyDropTable(
datatable: BenchmarkDatatableSeed,
sql: string
): Record<string, unknown>[] {
const match = sql.match(
/^\s*drop\s+table\s+(?:if\s+exists\s+)?([a-zA-Z_"][\w."]*)/i
)
if (!match) {
return []
}
const { schema, table } = parseRef(match[1])
if (datatable.schemas[schema]?.[table]) {
delete datatable.schemas[schema][table]
}
return []
}
// ============= DML =============
function applyInsert(
datatable: BenchmarkDatatableSeed,
sql: string
): Record<string, unknown>[] {
const { body, returning } = splitOffReturning(sql)
const match = body.match(
/^\s*insert\s+into\s+([a-zA-Z_"][\w."]*)\s*(?:\(([^)]*)\))?\s*values\s*([\s\S]+)$/i
)
if (!match) {
return []
}
const table = resolveTable(datatable, match[1])
if (!table) {
return []
}
const columns = match[2]
? splitTopLevel(match[2]).map((entry) => unquoteIdentifier(entry.trim()))
: Object.keys(table.columns)
const inserted: Record<string, unknown>[] = []
for (const tuple of extractParenGroups(match[3])) {
const values = splitTopLevel(tuple).map((entry) => parseValue(entry))
const row: Record<string, unknown> = {}
columns.forEach((column, index) => {
row[column] = values[index]
})
inserted.push(row)
}
table.rows ??= []
table.rows.push(...inserted)
return returning ? inserted : []
}
function applyUpdate(
datatable: BenchmarkDatatableSeed,
sql: string
): Record<string, unknown>[] {
const { body, returning } = splitOffReturning(sql)
const match = body.match(/^\s*update\s+([a-zA-Z_"][\w."]*)\s+set\s+([\s\S]+)$/i)
if (!match) {
return []
}
const table = resolveTable(datatable, match[1])
if (!table) {
return []
}
let assignmentText = match[2]
let whereText: string | undefined
const whereMatch = maskForClauseScan(assignmentText).match(/\swhere\s/i)
if (whereMatch && whereMatch.index !== undefined) {
whereText = assignmentText.slice(whereMatch.index + whereMatch[0].length)
assignmentText = assignmentText.slice(0, whereMatch.index)
}
const predicates = parsePredicates(whereText)
if (predicates === null) {
return []
}
const assignments: Record<string, unknown> = {}
for (const entry of splitTopLevel(assignmentText)) {
const pair = entry.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/)
if (pair) {
assignments[lastIdentifier(pair[1])] = parseValue(pair[2])
}
}
const affected = (table.rows ?? []).filter((row) => rowMatches(row, predicates))
for (const row of affected) {
Object.assign(row, assignments)
}
return returning ? affected : []
}
function applyDelete(
datatable: BenchmarkDatatableSeed,
sql: string
): Record<string, unknown>[] {
const { body, returning } = splitOffReturning(sql)
const match = body.match(/^\s*delete\s+from\s+([a-zA-Z_"][\w."]*)\s*([\s\S]*)$/i)
if (!match) {
return []
}
const table = resolveTable(datatable, match[1])
if (!table) {
return []
}
const whereText = match[2].replace(/^\s*where\s+/i, '').trim() || undefined
const predicates = parsePredicates(whereText)
if (predicates === null) {
return []
}
const rows = table.rows ?? []
const removed = rows.filter((row) => rowMatches(row, predicates))
table.rows = rows.filter((row) => !rowMatches(row, predicates))
return returning ? removed : []
}
// ============= Parsing helpers =============
function resolveTable(
datatable: BenchmarkDatatableSeed,
ref: string
): BenchmarkDatatableTableSeed | undefined {
const { schema, table } = parseRef(ref)
const direct = datatable.schemas[schema]?.[table]
if (direct) {
return direct
}
// Bare table name: fall back to searching every schema for a matching table.
if (!ref.includes('.')) {
for (const tables of Object.values(datatable.schemas)) {
if (tables[table]) {
return tables[table]
}
}
}
return undefined
}
function firstTable(
datatable: BenchmarkDatatableSeed
): BenchmarkDatatableTableSeed | undefined {
for (const tables of Object.values(datatable.schemas)) {
for (const seed of Object.values(tables)) {
return seed
}
}
return undefined
}
function parseRef(ref: string): ParsedRef {
const parts = ref.split('.').map(unquoteIdentifier)
if (parts.length >= 2) {
return { schema: parts[parts.length - 2], table: parts[parts.length - 1] }
}
return { schema: DEFAULT_SCHEMA, table: parts[0] }
}
/** A WHERE clause with no parseable form returns `null`; absent WHERE returns `[]` (match all). */
function parsePredicates(whereText: string | undefined): Predicate[] | null {
if (whereText === undefined || whereText.trim() === '') {
return []
}
const predicates: Predicate[] = []
for (const part of whereText.split(/\s+and\s+/i)) {
const match = part.match(/^\s*([a-zA-Z_"][\w."]*)\s*=\s*([\s\S]+?)\s*$/)
if (!match) {
return null
}
predicates.push({ column: lastIdentifier(match[1]), value: parseValue(match[2]) })
}
return predicates
}
function rowMatches(row: Record<string, unknown>, predicates: Predicate[]): boolean {
return predicates.every((predicate) => looseEquals(row[predicate.column], predicate.value))
}
function looseEquals(left: unknown, right: unknown): boolean {
if (left === null || left === undefined) {
return right === null || right === undefined
}
if (typeof left === 'number' && typeof right === 'number') {
return left === right
}
return String(left) === String(right)
}
function parseValue(raw: string): unknown {
// Drop a trailing Postgres cast (e.g. `2::int4`) before interpreting the literal.
const token = raw.trim().replace(/::\s*[a-zA-Z_][\w]*(\([^)]*\))?\s*$/, '').trim()
const stringMatch = token.match(/^'([\s\S]*)'$/)
if (stringMatch) {
return stringMatch[1].replace(/''/g, "'")
}
if (/^-?\d+(\.\d+)?$/.test(token)) {
return Number(token)
}
if (/^true$/i.test(token)) {
return true
}
if (/^false$/i.test(token)) {
return false
}
if (/^null$/i.test(token)) {
return null
}
return token
}
function splitOffReturning(sql: string): { body: string; returning: boolean } {
const match = maskForClauseScan(sql).match(/\sreturning\s/i)
if (!match || match.index === undefined) {
return { body: sql, returning: false }
}
return { body: sql.slice(0, match.index), returning: true }
}
/**
* A same-length copy of `sql` with the contents of single-quoted strings and
* parenthesized groups blanked to spaces, so a top-level keyword scan
* (WHERE / RETURNING) cannot match inside a string literal or a subquery. Index
* positions in the result map 1:1 back onto the original.
*/
function maskForClauseScan(sql: string): string {
let masked = ''
let depth = 0
let inString = false
for (let i = 0; i < sql.length; i++) {
const char = sql[i]
if (inString) {
if (char === "'") {
if (sql[i + 1] === "'") {
masked += ' '
i++
continue
}
inString = false
}
masked += ' '
continue
}
if (char === "'") {
inString = true
masked += ' '
} else if (char === '(') {
depth++
masked += ' '
} else if (char === ')') {
depth = Math.max(0, depth - 1)
masked += ' '
} else {
masked += depth > 0 ? ' ' : char
}
}
return masked
}
/**
* Inner text of each top-level `( ... )` group in `input`, honoring nested parens
* (e.g. `now()`, `numeric(10,2)`) and single-quoted strings. Used for the CREATE
* column-definition group and INSERT value tuples.
*/
function extractParenGroups(input: string): string[] {
const groups: string[] = []
let depth = 0
let inString = false
let current = ''
for (let i = 0; i < input.length; i++) {
const char = input[i]
if (inString) {
current += char
if (char === "'") {
if (input[i + 1] === "'") {
current += input[++i]
} else {
inString = false
}
}
continue
}
if (char === "'") {
inString = true
current += char
} else if (char === '(') {
depth++
if (depth === 1) {
current = ''
} else {
current += char
}
} else if (char === ')') {
depth = Math.max(0, depth - 1)
if (depth === 0) {
groups.push(current)
current = ''
} else {
current += char
}
} else if (depth > 0) {
current += char
}
}
return groups
}
/** Split on commas that are not inside parentheses or single-quoted strings. */
function splitTopLevel(input: string): string[] {
const parts: string[] = []
let depth = 0
let inString = false
let current = ''
for (let i = 0; i < input.length; i++) {
const char = input[i]
if (inString) {
current += char
if (char === "'") {
if (input[i + 1] === "'") {
current += input[++i]
} else {
inString = false
}
}
continue
}
if (char === "'") {
inString = true
current += char
} else if (char === '(') {
depth++
current += char
} else if (char === ')') {
depth = Math.max(0, depth - 1)
current += char
} else if (char === ',' && depth === 0) {
parts.push(current)
current = ''
} else {
current += char
}
}
if (current.trim() !== '') {
parts.push(current)
}
return parts
}
function isTableConstraint(def: string): boolean {
return /^(primary\s+key|foreign\s+key|constraint|unique|check|exclude|like)\b/i.test(def)
}
function unquoteIdentifier(identifier: string): string {
const trimmed = identifier.trim()
const quoted = trimmed.match(/^"([\s\S]*)"$/)
return quoted ? quoted[1] : trimmed
}
/** For a qualified reference like `orders.id`, keep only the final identifier. */
function lastIdentifier(reference: string): string {
const parts = reference.split('.')
return unquoteIdentifier(parts[parts.length - 1])
}
function stripTrailingSemicolon(sql: string): string {
return sql.replace(/;\s*$/, '')
}
ai_evals/adapters/frontend/mockBackend.ts
+108 -2
View File
@@ -1,7 +1,14 @@
import { randomUUID } from 'node:crypto'
import type { CompletedJob, Flow, Script } from '../../../frontend/src/lib/gen'
import type { ScriptLang } from '../../../frontend/src/lib/gen/types.gen'
import type {
DataTableTables,
DataTableTableSchema,
ScriptLang
} from '../../../frontend/src/lib/gen/types.gen'
import { buildScriptLintResult } from './core/script/preview'
import { applyDatatableSql, type BenchmarkDatatableSeed } from './datatableSqlEngine'
export type { BenchmarkDatatableSeed, BenchmarkDatatableTableSeed } from './datatableSqlEngine'
const BENCHMARK_TIMESTAMP = '1970-01-01T00:00:00.000Z'
@@ -25,6 +32,7 @@ export interface BenchmarkWorkspaceFlow {
export interface BenchmarkWorkspaceRunnables {
scripts?: BenchmarkWorkspaceScript[]
flows?: BenchmarkWorkspaceFlow[]
datatables?: BenchmarkDatatableSeed[]
}
type BenchmarkCompletedJob = CompletedJob & { type: 'CompletedJob' }
@@ -48,7 +56,12 @@ export function registerBenchmarkWorkspaceRunnables(
runnables: BenchmarkWorkspaceRunnables
): void {
benchmarkWorkspaces.add(workspace)
benchmarkWorkspaceRunnables.set(workspace, runnables)
// Datatables are mutated in place by exec_datatable_sql (a write must be visible
// to later reads), so store an isolated deep copy — never mutate the caller's seed.
benchmarkWorkspaceRunnables.set(workspace, {
...runnables,
datatables: runnables.datatables ? structuredClone(runnables.datatables) : undefined
})
}
export function unregisterBenchmarkWorkspace(workspace: string): void {
@@ -161,6 +174,99 @@ export function getBenchmarkCompletedJob(
return structuredClone(entry.job)
}
// ============= Datatables (best-effort in-memory SQL) =============
/**
* Project the seeded datatables down to the `list_datatable_tables` response:
* `datatable_name` + `schema -> table_names`, with no column detail.
* Returns `null` for a non-benchmark workspace so callers can fall through to
* the real backend; an empty seed yields `[]`.
*/
export function listBenchmarkDatatables(workspace: string): DataTableTables[] | null {
const runnables = benchmarkWorkspaceRunnables.get(workspace)
if (!runnables) {
return null
}
return (runnables.datatables ?? []).map((datatable) => ({
datatable_name: datatable.datatable_name,
schemas: Object.fromEntries(
Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)])
)
}))
}
export function getBenchmarkDatatableSchema(input: {
workspace: string
datatableName: string
schemaName: string
tableName: string
}): DataTableTableSchema {
const runnables = benchmarkWorkspaceRunnables.get(input.workspace)
const datatable = (runnables?.datatables ?? []).find(
(entry) => entry.datatable_name === input.datatableName
)
if (!datatable) {
// Message MUST match the production `isDatatableNotConfiguredError` regex
// (/datatable\s+\S+\s+not found/i in datatableTools.ts) so the
// get_datatable_table_schema not-configured mapping is actually exercised.
throw new Error(`datatable "${input.datatableName}" not found`)
}
const table = datatable.schemas?.[input.schemaName]?.[input.tableName]
if (!table) {
throw new Error(
`table "${input.schemaName}.${input.tableName}" not found in datatable "${input.datatableName}"`
)
}
return {
datatable_name: input.datatableName,
schema_name: input.schemaName,
table_name: input.tableName,
columns: table.columns
}
}
/**
* Execute SQL against a seeded datatable through the best-effort in-memory engine
* (`applyDatatableSql`). Writes (CREATE/INSERT/UPDATE/DELETE/DROP) mutate the
* stored datatable in place so a later list/schema/SELECT reflects them; SELECT
* (and RETURNING) yield rows, other statements yield `[]`. Creates a benchmark
* completed job and returns its id, like `runBenchmarkScriptPreview`.
*/
export function runBenchmarkDatatableSql(input: {
workspace: string
datatableName: string
sql: string
}): string {
const runnables = benchmarkWorkspaceRunnables.get(input.workspace)
const datatable = (runnables?.datatables ?? []).find(
(entry) => entry.datatable_name === input.datatableName
)
const rows = datatable ? applyDatatableSql(datatable, input.sql).rows : []
return createBenchmarkCompletedJob({
workspace: input.workspace,
jobKind: 'preview',
success: true,
args: { database: `datatable://${input.datatableName}` },
result: rows
})
}
/**
* Mirror `JobService.getCompletedJobResultMaybe` for benchmark workspaces — the
* shape `pollJobResult` consumes. The job is created synchronously before
* polling, so it is always present and completed.
*/
export function getBenchmarkCompletedJobResultMaybe(input: {
workspace: string
id: string
}): { success: boolean; completed: boolean; result: unknown } {
const job = getBenchmarkCompletedJob(input.workspace, input.id)
if (!job) {
throw new Error(`Job "${input.id}" not found in benchmark workspace`)
}
return { success: job.success, completed: true, result: job.result }
}
export function runBenchmarkScriptPreview(input: {
workspace: string
requestBody: {
ai_evals/adapters/frontend/mockBackendDatatables.test.ts
+175
View File
@@ -0,0 +1,175 @@
import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
import {
getBenchmarkCompletedJobResultMaybe,
getBenchmarkDatatableSchema,
listBenchmarkDatatables,
registerBenchmarkWorkspaceRunnables,
resetBenchmarkMockBackend,
runBenchmarkDatatableSql,
type BenchmarkWorkspaceRunnables
} from './mockBackend'
const WORKSPACE = 'benchmark-datatable-ws'
// Mirrors the production `isDatatableNotConfiguredError` regex in
// datatableTools.ts. The schema mock's "not configured" message MUST match it,
// otherwise the not-configured mapping in get_datatable_table_schema is silently
// untested.
const NOT_CONFIGURED_RE = /datatable\s+\S+\s+not found/i
const SEED: BenchmarkWorkspaceRunnables = {
datatables: [
{
datatable_name: 'main',
schemas: {
public: {
orders: {
columns: { id: 'int', total: 'numeric' },
rows: [
{ id: 1, total: 10 },
{ id: 2, total: 20 }
]
},
customers: {
columns: { id: 'int', name: 'text' },
rows: [{ id: 1, name: 'alice' }]
}
}
}
}
]
}
beforeEach(() => resetBenchmarkMockBackend())
afterEach(() => resetBenchmarkMockBackend())
describe('listBenchmarkDatatables', () => {
it('returns null for a non-benchmark workspace (caller falls through to real backend)', () => {
expect(listBenchmarkDatatables('unregistered')).toBeNull()
})
it('returns [] for a registered workspace with no datatables seed', () => {
registerBenchmarkWorkspaceRunnables(WORKSPACE, {})
expect(listBenchmarkDatatables(WORKSPACE)).toEqual([])
})
it('projects seeded datatables to schema -> table names only (no columns)', () => {
registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED)
expect(listBenchmarkDatatables(WORKSPACE)).toEqual([
{ datatable_name: 'main', schemas: { public: ['orders', 'customers'] } }
])
})
})
describe('getBenchmarkDatatableSchema', () => {
beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED))
it('returns the columns for a seeded table', () => {
expect(
getBenchmarkDatatableSchema({
workspace: WORKSPACE,
datatableName: 'main',
schemaName: 'public',
tableName: 'orders'
})
).toEqual({
datatable_name: 'main',
schema_name: 'public',
table_name: 'orders',
columns: { id: 'int', total: 'numeric' }
})
})
it('throws a not-configured error matching the production regex for an unknown datatable', () => {
let error: Error | undefined
try {
getBenchmarkDatatableSchema({
workspace: WORKSPACE,
datatableName: 'ghost',
schemaName: 'public',
tableName: 'orders'
})
} catch (e) {
error = e as Error
}
expect(error).toBeDefined()
expect(error!.message).toMatch(NOT_CONFIGURED_RE)
})
it('throws a table-not-found error that does NOT match the datatable-not-configured regex', () => {
// The datatable IS configured; only the table is missing. Production maps
// this to a generic "error getting schema", not the blocking message.
let error: Error | undefined
try {
getBenchmarkDatatableSchema({
workspace: WORKSPACE,
datatableName: 'main',
schemaName: 'public',
tableName: 'ghost'
})
} catch (e) {
error = e as Error
}
expect(error).toBeDefined()
expect(error!.message).not.toMatch(NOT_CONFIGURED_RE)
})
})
describe('runBenchmarkDatatableSql + getBenchmarkCompletedJobResultMaybe', () => {
beforeEach(() => registerBenchmarkWorkspaceRunnables(WORKSPACE, SEED))
function exec(sql: string): { success: boolean; completed: boolean; result: unknown } {
const jobId = runBenchmarkDatatableSql({ workspace: WORKSPACE, datatableName: 'main', sql })
return getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: jobId })
}
it('returns the canned rows of the table named in a SELECT FROM clause', () => {
expect(exec('SELECT * FROM customers')).toEqual({
success: true,
completed: true,
result: [{ id: 1, name: 'alice' }]
})
})
it('falls back to the first seeded table when the SELECT references no known table', () => {
expect(exec('select 1').result).toEqual([
{ id: 1, total: 10 },
{ id: 2, total: 20 }
])
})
it('returns [] success for DDL and DML statements without RETURNING', () => {
expect(exec('CREATE TABLE foo (id int)').result).toEqual([])
expect(exec('INSERT INTO orders VALUES (3, 30)').result).toEqual([])
expect(exec('update orders set total = 0').result).toEqual([])
})
it('reflects a write in a later SELECT, isolated from the shared seed', () => {
exec('UPDATE orders SET total = 999 WHERE id = 1')
expect((exec('SELECT * FROM orders').result as Record<string, unknown>[])).toContainEqual({
id: 1,
total: 999
})
// Registration deep-clones the seed, so the shared SEED const stays pristine.
expect(SEED.datatables![0].schemas.public.orders.rows).toContainEqual({ id: 1, total: 10 })
})
it('reflects a CREATE in list_datatables and get_datatable_table_schema', () => {
exec('CREATE TABLE public.refunds (order_id int4, amount numeric)')
expect(listBenchmarkDatatables(WORKSPACE)?.[0].schemas.public).toContain('refunds')
expect(
getBenchmarkDatatableSchema({
workspace: WORKSPACE,
datatableName: 'main',
schemaName: 'public',
tableName: 'refunds'
}).columns
).toEqual({ order_id: 'int4', amount: 'numeric' })
})
it('throws for an unknown job id', () => {
expect(() =>
getBenchmarkCompletedJobResultMaybe({ workspace: WORKSPACE, id: 'does-not-exist' })
).toThrow()
})
})
ai_evals/adapters/frontend/vitestAdapter.test.ts
+80 -8
View File
@@ -34,15 +34,19 @@ vi.mock('$lib/gen', async () => {
const actual = await vi.importActual<any>('$lib/gen')
const {
getBenchmarkCompletedJob,
getBenchmarkCompletedJobResultMaybe,
getBenchmarkDatatableSchema,
getBenchmarkFlowByPath,
getBenchmarkScriptByHash,
getBenchmarkScriptByPath,
hasBenchmarkWorkspace,
listBenchmarkDatatables,
listBenchmarkFlows,
listBenchmarkScripts,
createBenchmarkHttpTrigger,
createBenchmarkSchedule,
previewBenchmarkSchedule,
runBenchmarkDatatableSql,
runBenchmarkFlowByPath,
runBenchmarkScriptPreview
} = await import('./mockBackend')
@@ -79,6 +83,16 @@ vi.mock('$lib/gen', async () => {
}
return actual.ScriptService.getScriptByPath(data)
},
getScriptByPathWithDraft: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const script = getBenchmarkScriptByPath(data.workspace, data.path)
if (!script) {
throw new Error(`Script "${data.path}" not found in benchmark workspace`)
}
return script
}
return actual.ScriptService.getScriptByPathWithDraft(data)
},
getScriptByHash: async (data: { workspace: string; hash: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const script = getBenchmarkScriptByHash(data.workspace, data.hash)
@@ -108,6 +122,26 @@ vi.mock('$lib/gen', async () => {
return flow
}
return actual.FlowService.getFlowByPath(data)
},
getFlowByPathWithDraft: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
if (!flow) {
throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
}
return flow
}
return actual.FlowService.getFlowByPathWithDraft(data)
},
getFlowLatestVersion: async (data: { workspace: string; path: string }) => {
if (hasBenchmarkWorkspace(data.workspace)) {
const flow = getBenchmarkFlowByPath(data.workspace, data.path)
if (!flow) {
throw new Error(`Flow "${data.path}" not found in benchmark workspace`)
}
return { id: 1 }
}
return actual.FlowService.getFlowLatestVersion(data)
}
}),
JobService: wrapService(actual.JobService, {
@@ -119,13 +153,27 @@ vi.mock('$lib/gen', async () => {
args?: Record<string, unknown>
path?: string
}
}) =>
hasBenchmarkWorkspace(data.workspace)
? runBenchmarkScriptPreview({
workspace: data.workspace,
requestBody: data.requestBody ?? {}
})
: actual.JobService.runScriptPreview(data),
}) => {
if (!hasBenchmarkWorkspace(data.workspace)) {
return actual.JobService.runScriptPreview(data)
}
const requestBody = data.requestBody ?? {}
const database = requestBody.args?.database
// Datatable SQL runs as a `postgresql` preview against `datatable://<name>`.
// Execute it through the canned-SQL mock instead of linting it as a script.
if (
requestBody.language === 'postgresql' &&
typeof database === 'string' &&
database.startsWith('datatable://')
) {
return runBenchmarkDatatableSql({
workspace: data.workspace,
datatableName: database.slice('datatable://'.length),
sql: requestBody.content ?? ''
})
}
return runBenchmarkScriptPreview({ workspace: data.workspace, requestBody })
},
runFlowByPath: async (data: {
workspace: string
path: string
@@ -147,7 +195,31 @@ vi.mock('$lib/gen', async () => {
return job
}
return actual.JobService.getJob(data)
}
},
getCompletedJobResultMaybe: async (data: { workspace: string; id: string }) =>
hasBenchmarkWorkspace(data.workspace)
? getBenchmarkCompletedJobResultMaybe({ workspace: data.workspace, id: data.id })
: actual.JobService.getCompletedJobResultMaybe(data)
}),
WorkspaceService: wrapService(actual.WorkspaceService, {
listDataTableTables: async (data: { workspace: string }) =>
hasBenchmarkWorkspace(data.workspace)
? (listBenchmarkDatatables(data.workspace) ?? [])
: actual.WorkspaceService.listDataTableTables(data),
getDataTableTableSchema: async (data: {
workspace: string
datatableName: string
schemaName: string
tableName: string
}) =>
hasBenchmarkWorkspace(data.workspace)
? getBenchmarkDatatableSchema({
workspace: data.workspace,
datatableName: data.datatableName,
schemaName: data.schemaName,
tableName: data.tableName
})
: actual.WorkspaceService.getDataTableTableSchema(data)
}),
ScheduleService: wrapService(actual.ScheduleService, {
existsSchedule: async (data: { workspace: string; path: string }) =>
ai_evals/cases/flow.yaml
+11
View File
@@ -8,6 +8,9 @@
args:
a: 4
b: 5
toolExpect:
requiredToolsUsed:
- test_run_flow
judgeChecklist:
- "the flow takes `a` and `b` as inputs"
- "the main step is named `sum_numbers`"
@@ -25,6 +28,9 @@
args:
a: 2
b: 3
toolExpect:
requiredToolsUsed:
- test_run_flow
judgeChecklist:
- "the flow takes `a` and `b` as inputs"
- "the main step is named `sum_numbers`"
@@ -42,6 +48,9 @@
args:
a: 7
b: 8
toolExpect:
requiredToolsUsed:
- test_run_flow
judgeChecklist:
- "the parent flow takes `a` and `b` as inputs"
- "the main step is named `call_add_numbers`"
@@ -426,6 +435,7 @@
- return_schedule_status
toolExpect:
requiredToolsUsed:
- test_run_flow
- create_schedule
toolCallArgs:
- tool: create_schedule
@@ -453,6 +463,7 @@
- webhook_response
toolExpect:
requiredToolsUsed:
- test_run_flow
- create_trigger
toolCallArgs:
- tool: create_trigger
ai_evals/cases/global.yaml
+734
View File
@@ -87,3 +87,737 @@
- the flow accepts numeric inputs a and b
- the flow returns the sum of a and b
- the result stays as an AI draft and is not deployed or saved to the workspace
- id: global-test4-multi-artifact-notification-job
prompt: |-
Set up a draft stale-trial notification job.
Create a Bun script at `f/evals/global/check_stale_trials` that accepts `max_age_days`, uses mocked inline trial account data, and returns the stale trial account IDs.
Also create a weekday 09:00 UTC schedule at `f/evals/global/check_stale_trials_weekday` for that script with `max_age_days` set to 14.
Add an HTTP POST trigger at `f/evals/global/check_stale_trials_manual` with route path `evals/check-stale-trials` that runs the same script manually.
Leave everything as AI drafts only; do not deploy or save anything to the workspace.
runtime:
maxTurns: 12
validate:
draftCountExactly: 3
requiredDrafts:
- type: script
path: f/evals/global/check_stale_trials
language: bun
valueIncludes:
- max_age_days
- trial
- type: schedule
path: f/evals/global/check_stale_trials_weekday
valueIncludes:
- f/evals/global/check_stale_trials
- UTC
- "14"
- type: trigger
triggerKind: http
path: f/evals/global/check_stale_trials_manual
valueIncludes:
- evals/check-stale-trials
- f/evals/global/check_stale_trials
toolExpect:
requiredToolsUsed:
- write_script
- write_schedule
- write_trigger
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates a Bun script draft for stale trial accounts
- creates a weekday 09:00 UTC schedule draft for the script with max_age_days set to 14
- creates an HTTP POST trigger draft with route path evals/check-stale-trials for the same script
- leaves all artifacts as drafts only and does not deploy
- id: global-test5-existing-flow-inline-code-edit
prompt: |-
Update the existing flow at `f/evals/global/process_invoice`.
Only change the `calculate_total` inline code so it applies 8% tax and returns an object containing `subtotal`, `tax`, and `total`.
Leave the updated flow as an AI draft only; do not deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
path: f/evals/global/process_invoice
valueIncludes:
- calculate_total
- tax
- total
toolExpect:
requiredToolsUsed:
- read_workspace_item
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- reads the existing process_invoice flow before editing it
- updates the calculate_total inline code to apply 8% tax
- returns subtotal, tax, and total from the updated flow logic
- leaves the result as an AI draft only
- id: global-test6-secret-variable-draft
prompt: |-
Create a secret variable draft at `f/evals/global/slack_bot_token`.
Use the placeholder value `xoxb-redacted-test-token` and description `Slack bot token for eval notifications`.
Do not create any resource or deploy anything.
runtime:
maxTurns: 6
validate:
draftCountExactly: 1
requiredDrafts:
- type: variable
path: f/evals/global/slack_bot_token
valueIncludes:
- Slack bot token
- "true"
forbiddenDrafts:
- type: resource
path: f/evals/global/slack_bot_token
toolExpect:
requiredToolsUsed:
- write_variable
forbiddenToolsUsed:
- write_resource
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: write_variable
field: value
stringStartsWithAnyOf:
- xoxb-redacted-test-token
skipJudge: true
judgeChecklist:
- creates exactly one secret variable draft at f/evals/global/slack_bot_token
- uses the requested placeholder value and description
- does not create a resource or deploy anything
- id: global-test7-ambiguous-app-asks-question
prompt: |-
Create a new raw app for triaging support tickets.
runtime:
maxTurns: 4
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- askUserQuestion
forbiddenToolsUsed:
- init_app
- write_app_file
- write_app_runnable
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
- id: global-test8-human-script-infer-path-language
prompt: |-
I need a small helper that formats a customer-facing welcome line.
It should take a person's name and return "Welcome aboard, <name>!".
Please just stage it as a draft for now.
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
valueIncludes:
- Welcome aboard
- name
toolExpect:
requiredToolsUsed:
- write_script
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates a single script draft for a welcome-line helper
- accepts a person's name as input
- returns a message containing Welcome aboard, the provided name, and an exclamation mark
- chooses a reasonable workspace path and script language without needing the user to specify them
- leaves the result as an AI draft only
- id: global-test9-human-weekday-trial-job
prompt: |-
Can you set up a draft daily job that checks a few hard-coded trial accounts and returns the ones whose trial has ended?
It should run every weekday morning around 9 in UTC with a 30 day cutoff.
Keep it as draft work only.
runtime:
maxTurns: 10
validate:
draftCountExactly: 2
requiredDrafts:
- type: script
pathIncludes:
- trial
valueIncludes:
- trial
- "30"
- type: schedule
pathIncludes:
- trial
valueIncludes:
- UTC
toolExpect:
requiredToolsUsed:
- write_script
- write_schedule
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates a script draft that checks hard-coded trial accounts
- returns the accounts whose trial has ended based on a 30 day cutoff
- creates a schedule draft for weekday mornings around 09:00 UTC
- links the schedule to the generated script
- leaves both artifacts as drafts only
- id: global-test10-human-secret-variable
prompt: |-
I need a placeholder Slack bot token stored securely for future notification work.
Use xoxb-redacted-test-token and note that it is for eval notifications.
Only prepare a draft.
runtime:
maxTurns: 6
validate:
draftCountExactly: 1
requiredDrafts:
- type: variable
pathIncludes:
- slack
valueIncludes:
- eval notifications
- "true"
toolExpect:
requiredToolsUsed:
- write_variable
forbiddenToolsUsed:
- write_resource
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: write_variable
field: value
stringStartsWithAnyOf:
- xoxb-redacted-test-token
skipJudge: true
judgeChecklist:
- creates a single secret variable draft for the Slack bot token placeholder
- uses the requested placeholder value
- includes a note or description that it is for eval notifications
- does not create a resource or deploy anything
- id: global-test11-human-existing-flow-informal-edit
prompt: |-
There is an invoice processing flow in this workspace.
Can you adjust its total calculation so it adds 8% tax and returns subtotal, tax, and total?
Keep the change as a draft.
initial: ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
pathIncludes:
- invoice
valueIncludes:
- calculate_total
- tax
- total
toolExpect:
requiredToolsUsed:
- read_workspace_item
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- finds and edits the existing invoice processing flow without the user providing its exact path
- updates the total calculation to apply 8% tax
- returns subtotal, tax, and total from the updated flow logic
- leaves the result as an AI draft only
- id: global-test12-current-live-script-edit
prompt: |-
The script I have open formats greetings.
Can you update this script so it uppercases the name before greeting them and ends with an exclamation mark?
Keep it as draft work.
initial: ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
path: f/evals/global/current_greeting
language: bun
valueIncludes:
- toUpperCase
- "!"
forbiddenDrafts:
- type: script
path: f/evals/global/format_greeting
- type: script
path: f/evals/global/format_greeting_archive
toolExpect:
requiredToolsUsed:
- read_workspace_item
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- resolves "this script" to the active live editor script instead of another similarly named workspace script
- updates the greeting logic to uppercase the provided name
- returns a greeting ending with an exclamation mark
- leaves the result as a draft only
- id: global-test13-current-live-flow-edit
prompt: |-
I have the invoice flow open.
In the current flow, update the total calculation to add 8% tax and return subtotal, tax, and total.
Keep the change as a draft.
initial: ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: flow
path: f/evals/global/current_invoice_flow
valueIncludes:
- calculate_total
- tax
- total
forbiddenDrafts:
- type: flow
path: f/evals/global/process_invoice
- type: flow
path: f/evals/global/process_refund
toolExpect:
requiredToolsUsed:
- read_workspace_item
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- resolves "current flow" to the active live editor flow
- does not edit the similarly named deployed invoice or refund flows
- updates the calculate_total logic to apply 8% tax
- returns subtotal, tax, and total from the updated flow logic
- leaves the result as a draft only
- id: global-test14-current-without-live-editor-asks-question
prompt: |-
Please update this script so it returns `ok`.
Keep it as a draft.
runtime:
maxTurns: 4
validate:
draftCountExactly: 0
toolExpect:
forbiddenToolsUsed:
- write_script
- edit_script
- write_flow
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- asks which script to update when the user refers to "this script" without selected or active editor context
- does not guess a path or create a new script draft
- id: global-test15-human-postgres-resource
prompt: |-
I'm wiring the eval reporting database into this workspace.
Can you stage a Postgres connection for it in the shared evals/global folder?
Use host `reports-db.internal`, port 5432, database `evals_reporting`, user `report_reader`, and password `pg-redacted-reporting-password`.
Keep the credentials safe.
This is just draft work for now.
runtime:
maxTurns: 10
validate:
draftCountExactly: 2
requiredDrafts:
- type: variable
pathStartsWith: f/evals/global/
pathIncludes:
- evals
- global
- report
- password
valueIncludes:
- "true"
- report
- type: resource
pathStartsWith: f/evals/global/
pathIncludes:
- evals
- global
- report
valueIncludes:
- postgres
- reports-db.internal
- "5432"
- evals_reporting
- report_reader
- "$var:"
valueExcludes:
- pg-redacted-reporting-password
toolExpect:
requiredToolsUsed:
- write_variable
- search_resource_types
- write_resource
forbiddenToolsUsed:
- write_schedule
- write_trigger
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: write_variable
field: value
stringStartsWithAnyOf:
- pg-redacted-reporting-password
skipJudge: true
judgeChecklist:
- creates a Postgres resource draft for the eval reporting database
- creates a secret variable draft for the database password
- puts the drafts in sensible eval/global reporting-related paths
- uses the requested host, port, database, and user
- references the secret variable from the resource instead of embedding the password
- leaves the work as a draft only
- id: global-test16-human-visible-variable
prompt: |-
We keep reusing a 30 day trial cutoff in eval notification jobs.
Can you stage that as a normal workspace variable in the shared evals/global folder, with a short description so people know what it controls?
It is not a secret.
runtime:
maxTurns: 6
validate:
draftCountExactly: 1
requiredDrafts:
- type: variable
pathStartsWith: f/evals/global/
pathIncludes:
- evals
- global
- trial
valueIncludes:
- "30"
- "false"
- trial
toolExpect:
requiredToolsUsed:
- write_variable
forbiddenToolsUsed:
- write_resource
- write_schedule
- write_trigger
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- creates exactly one non-secret variable draft for the trial cutoff
- stores the value 30
- chooses a sensible eval/global path related to trials or notifications
- includes a useful description of what the value controls
- does not create resources, schedules, triggers, or deployed workspace changes
- id: global-test17-human-schedule-existing-helper
prompt: |-
The workspace already has a report digest helper.
Can you stage a weekday 8:30 AM UTC run for it with `dry_run` turned on?
I only want the schedule draft for review.
initial: ai_evals/fixtures/frontend/global/initial/report_digest_script.json
runtime:
maxTurns: 8
validate:
draftCountExactly: 1
requiredDrafts:
- type: schedule
pathIncludes:
- digest
valueIncludes:
- f/evals/global/send_report_digest
- UTC
- dry_run
- "true"
toolExpect:
requiredToolsUsed:
- list_workspace_items
- write_schedule
forbiddenToolsUsed:
- write_script
- write_flow
- write_resource
- write_variable
- write_trigger
- deploy_workspace_item
- delete_workspace_item
judgeChecklist:
- finds the existing report digest helper rather than creating a new script or flow
- creates one schedule draft for that helper
- schedules it for weekdays around 08:30 UTC
- passes dry_run as true
- leaves only the schedule draft for review
- id: global-test18-human-slack-resource-with-secret
prompt: |-
I'm preparing Slack notifications for eval failures.
Can you stage a Slack connection in the shared evals/global folder?
The bot token is `xoxb-redacted-test-token`; keep it safe.
Don't deploy anything yet.
runtime:
maxTurns: 8
validate:
draftCountExactly: 2
requiredDrafts:
- type: variable
pathStartsWith: f/evals/global/
pathIncludes:
- evals
- global
- slack
- token
valueIncludes:
- "true"
- type: resource
pathStartsWith: f/evals/global/
pathIncludes:
- evals
- global
- slack
valueIncludes:
- slack
- "$var:"
valueExcludes:
- xoxb-redacted-test-token
toolExpect:
requiredToolsUsed:
- write_variable
- search_resource_types
- write_resource
forbiddenToolsUsed:
- write_schedule
- write_trigger
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: write_variable
field: value
stringStartsWithAnyOf:
- xoxb-redacted-test-token
skipJudge: true
judgeChecklist:
- creates a secret variable draft for the Slack bot token placeholder
- creates a Slack resource draft that references the secret variable instead of embedding the token
- keeps both drafts under a sensible eval/global Slack-related path
- does not create schedules, triggers, or deployed workspace changes
- id: global-test19-datatable-not-configured-asks-to-set-up
prompt: |-
Here are two newsletter signups: alice@example.com and bob@example.com.
Save them into a workspace data table for me.
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- list_datatables
forbiddenToolsUsed:
- exec_datatable_sql
- write_script
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- checks which data tables exist in the workspace before acting
- recognizes that no data table is configured in this workspace
- explains that a data table must first be set up by the user in the workspace settings (Data Tables) and is not created via SQL
- does not run SQL, write a script, or invent a data table to work around the missing configuration
- tells the user to configure a data table and then try again
- id: global-test20-datatable-no-hallucinated-main
prompt: |-
Pull the latest rows from the orders table in our data table so I can see recent orders.
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- list_datatables
forbiddenToolsUsed:
- exec_datatable_sql
- write_script
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- checks which data tables exist in the workspace before querying
- recognizes that no data table is configured in this workspace
- does not assume a data table named "main" (or any other name) exists
- does not run SQL against a guessed data table or fabricate order rows
- tells the user they need to set up a data table in the workspace settings first
- id: global-test21-datatable-list-summarize
prompt: |-
What tables do we have in our workspace data table? Just give me the list.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- list_datatables
forbiddenToolsUsed:
- get_datatable_table_schema
- exec_datatable_sql
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- lists the tables available in the workspace data table (orders and customers)
- answers from the data table listing rather than fabricating table names
- does not fetch column details or run SQL just to produce a table list
- id: global-test22-datatable-inspect-columns
prompt: |-
What columns does the orders table have in our workspace data table?
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- get_datatable_table_schema
forbiddenToolsUsed:
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- inspects the orders table schema in the workspace data table
- reports the orders columns (such as id, customer_id, total, status, created_at)
- answers from the retrieved schema rather than guessing the columns
- id: global-test23-datatable-query-select
prompt: |-
Show me the orders in our workspace data table, including their status and total.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- exec_datatable_sql
forbiddenToolsUsed:
- write_script
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: exec_datatable_sql
field: sql
stringIncludesAnyOf:
- select
skipJudge: true
judgeChecklist:
- runs a SELECT query against the orders table in the workspace data table
- reports the orders returned by the query back to the user instead of fabricating data
- does not tell the user to set up a data table, since one already exists
- id: global-test24-datatable-create-table
prompt: |-
Add a new table called refunds to our workspace data table, with an order id and a refund amount.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 6
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- exec_datatable_sql
forbiddenToolsUsed:
- write_script
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: exec_datatable_sql
field: sql
stringIncludesAnyOf:
- create table
skipJudge: true
judgeChecklist:
- creates the refunds table with a plain CREATE TABLE statement on the data table
- includes an order id and a refund amount column
- treats creating the table as a normal SQL statement and does not claim a separate registration step is needed
- does not write a script to create the table
- id: global-test25-datatable-mutate-rows
prompt: |-
Mark order number 2 as shipped in our workspace data table.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
# Headroom for inspect-schema -> UPDATE -> verify; the in-memory engine now
# persists the write, so verification confirms on the first try (no loop).
maxTurns: 12
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- exec_datatable_sql
forbiddenToolsUsed:
- write_script
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: exec_datatable_sql
field: sql
stringIncludesAnyOf:
- update
- insert into
skipJudge: true
judgeChecklist:
- runs an UPDATE on the orders table setting the status of order id 2 to shipped
- targets only order number 2 rather than rewriting the whole table
- confirms the change back to the user
- id: global-test26-datatable-script-sdk
prompt: |-
Write a script that reads our workspace data table and returns the total revenue across all orders.
Leave it as an AI draft only; do not deploy or save it.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
runtime:
maxTurns: 10
validate:
draftCountExactly: 1
requiredDrafts:
- type: script
valueIncludes:
- wmill.datatable(
toolExpect:
requiredToolsUsed:
- get_instructions
- write_script
forbiddenToolsUsed:
- exec_datatable_sql
- deploy_workspace_item
- delete_workspace_item
# The judge has no datatable SDK reference and wrongly penalizes correct
# wmill.datatable() tagged-template usage, so rely on the deterministic checks:
# required get_instructions + write_script, forbidden exec_datatable_sql, and a
# draft that contains wmill.datatable(.
skipJudge: true
judgeChecklist:
- writes a script (not a chat-time SQL execution) that reads the workspace data table at runtime
- uses the wmill.datatable() SDK to query the orders table and sum the order totals
- returns the total revenue from the script
- leaves the result as an AI draft and does not deploy or save it
ai_evals/cases/script.yaml
+5
View File
@@ -5,6 +5,9 @@
Keep it simple and do not add external dependencies.
initial: ai_evals/fixtures/frontend/script/initial/test1_empty_bun.json
expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json
toolExpect:
requiredToolsUsed:
- test_run_script
judgeChecklist:
- uses the existing `name` input
- returns a plain greeting string
@@ -20,6 +23,7 @@
expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json
toolExpect:
requiredToolsUsed:
- test_run_script
- create_schedule
toolCallArgs:
- tool: create_schedule
@@ -44,6 +48,7 @@
expected: ai_evals/fixtures/frontend/script/expected/test1_greet_user.json
toolExpect:
requiredToolsUsed:
- test_run_script
- create_trigger
toolCallArgs:
- tool: create_trigger
ai_evals/cli/index.ts
+7 -3
View File
@@ -211,7 +211,7 @@ async function handleRun(input: {
const summaries: Array<{
label: string;
passRate: number;
averageDurationMs: number;
averagePassedDurationMs: number | null;
}> = [];
for (const [index, model] of models.entries()) {
@@ -259,7 +259,7 @@ async function handleRun(input: {
summaries.push({
label: `${model.id} (${runModel})`,
passRate: result.passRate,
averageDurationMs: result.averageDurationMs,
averagePassedDurationMs: result.averagePassedDurationMs ?? null,
});
}
@@ -267,7 +267,7 @@ async function handleRun(input: {
process.stdout.write("\nModel summary\n");
for (const summary of summaries) {
process.stdout.write(
`- ${summary.label}: ${formatPercent(summary.passRate)} | ${Math.round(summary.averageDurationMs)}ms\n`,
`- ${summary.label}: ${formatPercent(summary.passRate)} | passed avg ${formatNullableDuration(summary.averagePassedDurationMs)}\n`,
);
}
}
@@ -351,6 +351,10 @@ function formatPercent(value: number): string {
return `${(value * 100).toFixed(1)}%`;
}
function formatNullableDuration(value: number | null): string {
return value === null ? "n/a" : `${Math.round(value)}ms`;
}
void main().catch((error) => {
const message = error instanceof Error ? error.message : String(error);
process.stderr.write(`${message}\n`);
ai_evals/core/cases.test.ts
+44 -1
View File
@@ -14,6 +14,21 @@ describe("loadCases", () => {
},
},
});
expect(caseEntry?.toolExpect).toEqual({
requiredToolsUsed: ["test_run_flow"],
});
});
it("loads script and flow test tool expectations", async () => {
const scriptCases = await loadCases("script");
const flowCases = await loadCases("flow");
expect(scriptCases.find((entry) => entry.id === "script-test1-greet-user")?.toolExpect).toEqual({
requiredToolsUsed: ["test_run_script"],
});
expect(flowCases.find((entry) => entry.id === "flow-test0-sum-two-numbers")?.toolExpect).toEqual({
requiredToolsUsed: ["test_run_flow"],
});
});
it("loads the workspace-flow preference benchmark case", async () => {
@@ -203,6 +218,34 @@ describe("loadCases", () => {
});
});
it("loads global active-editor eval cases", async () => {
const globalCases = await loadCases("global");
const scriptCase = globalCases.find(
(entry) => entry.id === "global-test12-current-live-script-edit"
);
const flowCase = globalCases.find(
(entry) => entry.id === "global-test13-current-live-flow-edit"
);
expect(scriptCase?.initialPath).toContain(
"ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json"
);
expect(scriptCase?.toolExpect).toMatchObject({
requiredToolsUsed: ["read_workspace_item"],
});
expect(flowCase?.initialPath).toContain(
"ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json"
);
expect(flowCase?.validate).toMatchObject({
requiredDrafts: [
{
type: "flow",
path: "f/evals/global/current_invoice_flow",
},
],
});
});
it("loads tool expectations for workspace mutation cases", async () => {
const scriptCases = await loadCases("script");
const caseEntry = scriptCases.find(
@@ -210,7 +253,7 @@ describe("loadCases", () => {
);
expect(caseEntry?.toolExpect).toEqual({
requiredToolsUsed: ["create_schedule"],
requiredToolsUsed: ["test_run_script", "create_schedule"],
toolCallArgs: [
{
tool: "create_schedule",
ai_evals/core/models.test.ts
+17 -10
View File
@@ -2,15 +2,22 @@ import { describe, expect, it } from "bun:test";
import { resolveEvalModel } from "./models";
describe("resolveEvalModel", () => {
it("supports GPT-5.5 aliases for frontend evals", () => {
expect(resolveEvalModel("flow", "gpt-5.5").frontend).toEqual({
provider: "openai",
model: "gpt-5.5",
});
expect(resolveEvalModel("app", "gpt-55").frontend).toEqual({
provider: "openai",
model: "gpt-5.5",
});
expect(resolveEvalModel("script", "5.5").frontend).toEqual({
provider: "openai",
model: "gpt-5.5",
});
});
it("supports Gemini aliases for frontend evals", () => {
expect(resolveEvalModel("flow", "gemini").frontend).toEqual({
provider: "googleai",
model: "gemini-2.5-flash",
});
expect(resolveEvalModel("app", "gemini-pro").frontend).toEqual({
provider: "googleai",
model: "gemini-2.5-pro",
});
expect(
resolveEvalModel("script", "gemini-3-flash-preview").frontend,
).toEqual({
@@ -37,8 +44,8 @@ describe("resolveEvalModel", () => {
});
it("rejects Gemini aliases for cli evals", () => {
expect(() => resolveEvalModel("cli", "gemini")).toThrow(
"Model gemini-flash is not supported for cli mode",
expect(() => resolveEvalModel("cli", "gemini-3-flash-preview")).toThrow(
"Model gemini-3-flash-preview is not supported for cli mode",
);
});
});
ai_evals/core/models.ts
+5 -14
View File
@@ -88,21 +88,12 @@ export const EVAL_MODELS: EvalModelSpec[] = [
},
},
{
id: "gemini-flash",
label: "Gemini 2.5 Flash",
aliases: ["gemini", "gemini-flash", "gemini-2.5-flash"],
id: "gpt-5.5",
label: "GPT-5.5",
aliases: ["gpt-5.5", "gpt-55", "5.5"],
frontend: {
provider: "googleai",
model: "gemini-2.5-flash",
},
},
{
id: "gemini-pro",
label: "Gemini 2.5 Pro",
aliases: ["gemini-pro", "gemini-2.5-pro"],
frontend: {
provider: "googleai",
model: "gemini-2.5-pro",
provider: "openai",
model: "gpt-5.5",
},
},
{
ai_evals/core/results.test.ts
+242
View File
@@ -0,0 +1,242 @@
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { join } from "node:path";
import { tmpdir } from "node:os";
import { describe, expect, it } from "bun:test";
import {
appendHistoryRecord,
buildRunResult,
formatRunSummary,
} from "./results";
import type { BenchmarkCaseResult } from "./types";
function caseResult(
attempts: BenchmarkCaseResult["attempts"],
): BenchmarkCaseResult {
return {
id: "case-1",
prompt: "Do the thing",
attempts,
};
}
describe("benchmark results", () => {
it("keeps success cost metrics separate from failed attempts", () => {
const result = buildRunResult({
mode: "global",
runs: 1,
runModel: "model-under-test",
judgeModel: "judge-model",
caseResults: [
caseResult([
{
attempt: 1,
passed: true,
durationMs: 1000,
assistantMessageCount: 1,
toolCallCount: 1,
toolsUsed: ["edit_script"],
skillsInvoked: [],
checks: [{ name: "edited", passed: true }],
judgeScore: 100,
judgeSummary: "ok",
error: null,
tokenUsage: { prompt: 100, completion: 20, total: 120 },
},
{
attempt: 2,
passed: false,
durationMs: 100,
assistantMessageCount: 1,
toolCallCount: 0,
toolsUsed: [],
skillsInvoked: [],
checks: [{ name: "edited", passed: false }],
judgeScore: 10,
judgeSummary: "missed",
error: "failed",
tokenUsage: { prompt: 10, completion: 5, total: 15 },
},
]),
],
});
expect(result.attemptCount).toBe(2);
expect(result.passedAttempts).toBe(1);
expect(result.passRate).toBe(0.5);
expect(result.averageDurationMs).toBe(550);
expect(result.averagePassedDurationMs).toBe(1000);
expect(result.totalTokenUsage).toEqual({
prompt: 110,
completion: 25,
total: 135,
});
expect(result.totalPassedTokenUsage).toEqual({
prompt: 100,
completion: 20,
total: 120,
});
expect(result.averageTokenUsagePerAttempt).toEqual({
prompt: 55,
completion: 12.5,
total: 67.5,
});
expect(result.averageTokenUsagePerPassedAttempt).toEqual({
prompt: 100,
completion: 20,
total: 120,
});
const summary = formatRunSummary(result);
expect(summary).toContain("Average duration (passed): 1000ms");
expect(summary).toContain("Average tokens (passed): 120 total");
expect(summary).toContain("Average duration (all attempts): 550ms");
});
it("reports passed averages as unavailable when no attempt passes", () => {
const result = buildRunResult({
mode: "global",
runs: 1,
runModel: "model-under-test",
judgeModel: "judge-model",
caseResults: [
caseResult([
{
attempt: 1,
passed: false,
durationMs: 100,
assistantMessageCount: 1,
toolCallCount: 0,
toolsUsed: [],
skillsInvoked: [],
checks: [{ name: "edited", passed: false }],
judgeScore: 10,
judgeSummary: "missed",
error: "failed",
tokenUsage: { prompt: 10, completion: 5, total: 15 },
},
]),
],
});
expect(result.averagePassedDurationMs).toBeNull();
expect(result.totalPassedTokenUsage).toBeNull();
expect(result.averageTokenUsagePerPassedAttempt).toBeNull();
expect(formatRunSummary(result)).toContain(
"Average duration (passed): n/a",
);
});
it("normalizes passed token averages by passed attempts", () => {
const result = buildRunResult({
mode: "global",
runs: 1,
runModel: "model-under-test",
judgeModel: "judge-model",
caseResults: [
caseResult([
{
attempt: 1,
passed: true,
durationMs: 1000,
assistantMessageCount: 1,
toolCallCount: 1,
toolsUsed: ["edit_script"],
skillsInvoked: [],
checks: [{ name: "edited", passed: true }],
judgeScore: 100,
judgeSummary: "ok",
error: null,
tokenUsage: { prompt: 100, completion: 20, total: 120 },
},
{
attempt: 2,
passed: true,
durationMs: 1200,
assistantMessageCount: 1,
toolCallCount: 1,
toolsUsed: ["edit_script"],
skillsInvoked: [],
checks: [{ name: "edited", passed: true }],
judgeScore: 100,
judgeSummary: "ok",
error: null,
tokenUsage: null,
},
]),
],
});
expect(result.passedAttempts).toBe(2);
expect(result.totalPassedTokenUsage).toEqual({
prompt: 100,
completion: 20,
total: 120,
});
expect(result.averageTokenUsagePerPassedAttempt).toEqual({
prompt: 50,
completion: 10,
total: 60,
});
});
it("records passed-attempt metrics in history", async () => {
const tempDir = await mkdtemp(join(tmpdir(), "windmill-ai-evals-"));
try {
const historyPath = join(tempDir, "history.jsonl");
const result = buildRunResult({
mode: "global",
runs: 1,
runModel: "model-under-test",
judgeModel: "judge-model",
caseResults: [
caseResult([
{
attempt: 1,
passed: true,
durationMs: 1000,
assistantMessageCount: 1,
toolCallCount: 1,
toolsUsed: ["edit_script"],
skillsInvoked: [],
checks: [{ name: "edited", passed: true }],
judgeScore: 100,
judgeSummary: "ok",
error: null,
tokenUsage: { prompt: 100, completion: 20, total: 120 },
},
{
attempt: 2,
passed: false,
durationMs: 100,
assistantMessageCount: 1,
toolCallCount: 0,
toolsUsed: [],
skillsInvoked: [],
checks: [{ name: "edited", passed: false }],
judgeScore: 10,
judgeSummary: "missed",
error: "failed",
tokenUsage: { prompt: 10, completion: 5, total: 15 },
},
]),
],
});
await appendHistoryRecord(result, historyPath);
const record = JSON.parse(await readFile(historyPath, "utf8"));
expect(record.averageDurationMs).toBe(550);
expect(record.averagePassedDurationMs).toBe(1000);
expect(record.averageTokenUsagePerAttempt.total).toBe(67.5);
expect(record.averageTokenUsagePerPassedAttempt.total).toBe(120);
expect(record.cases[0].averageDurationMs).toBe(550);
expect(record.cases[0].averagePassedDurationMs).toBe(1000);
expect(record.cases[0].averageTokenUsagePerAttempt.total).toBe(67.5);
expect(record.cases[0].averageTokenUsagePerPassedAttempt.total).toBe(
120,
);
} finally {
await rm(tempDir, { recursive: true, force: true });
}
});
});
ai_evals/core/results.ts
+114 -67
View File
@@ -4,12 +4,20 @@ import { execFileSync } from "node:child_process";
import { getAiEvalsRoot, getRepoRoot } from "./cases";
import type {
BenchmarkArtifactFile,
BenchmarkAttemptResult,
BenchmarkCaseResult,
BenchmarkRunResult,
BenchmarkTokenUsage,
EvalMode,
} from "./types";
type AttemptAggregate = {
attemptCount: number;
durationTotal: number;
tokenUsageAttemptCount: number;
tokenUsageTotal: BenchmarkTokenUsage | null;
};
export async function writeRunResult(
result: BenchmarkRunResult,
outputPath?: string,
@@ -77,36 +85,12 @@ export function buildRunResult(input: {
judgeModel: string | null;
caseResults: BenchmarkCaseResult[];
}): BenchmarkRunResult {
const attemptCount = input.caseResults.reduce(
(sum, entry) => sum + entry.attempts.length,
0,
);
const passedAttempts = input.caseResults.reduce(
(sum, entry) =>
sum + entry.attempts.filter((attempt) => attempt.passed).length,
0,
);
const durationTotal = input.caseResults.reduce(
(sum, entry) =>
sum +
entry.attempts.reduce((inner, attempt) => inner + attempt.durationMs, 0),
0,
);
const tokenUsageTotal = input.caseResults.reduce<BenchmarkTokenUsage | null>(
(sum, entry) => {
for (const attempt of entry.attempts) {
if (!attempt.tokenUsage) {
continue;
}
sum ??= { prompt: 0, completion: 0, total: 0 };
sum.prompt += attempt.tokenUsage.prompt;
sum.completion += attempt.tokenUsage.completion;
sum.total += attempt.tokenUsage.total;
}
return sum;
},
null,
);
const attempts = input.caseResults.flatMap((entry) => entry.attempts);
const passedAttemptResults = attempts.filter((attempt) => attempt.passed);
const attemptAggregate = aggregateAttempts(attempts);
const passedAttemptAggregate = aggregateAttempts(passedAttemptResults);
const attemptCount = attemptAggregate.attemptCount;
const passedAttempts = passedAttemptAggregate.attemptCount;
return {
version: 1,
@@ -120,16 +104,19 @@ export function buildRunResult(input: {
attemptCount,
passedAttempts,
passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
averageDurationMs: attemptCount === 0 ? 0 : durationTotal / attemptCount,
totalTokenUsage: tokenUsageTotal,
averageDurationMs:
attemptCount === 0 ? 0 : attemptAggregate.durationTotal / attemptCount,
averagePassedDurationMs: averageDuration(passedAttemptAggregate),
totalTokenUsage: attemptAggregate.tokenUsageTotal,
totalPassedTokenUsage: passedAttemptAggregate.tokenUsageTotal,
averageTokenUsagePerAttempt:
attemptCount === 0 || !tokenUsageTotal
attemptCount === 0
? null
: {
prompt: tokenUsageTotal.prompt / attemptCount,
completion: tokenUsageTotal.completion / attemptCount,
total: tokenUsageTotal.total / attemptCount,
},
: averageTokenUsage(attemptAggregate, attemptCount),
averageTokenUsagePerPassedAttempt: averageTokenUsage(
passedAttemptAggregate,
passedAttempts,
),
cases: input.caseResults,
};
}
@@ -138,9 +125,25 @@ export function formatRunSummary(result: BenchmarkRunResult): string {
const lines = [
`${result.mode} benchmark complete`,
`Pass rate: ${formatPercent(result.passRate)} (${result.passedAttempts}/${result.attemptCount})`,
`Average duration: ${Math.round(result.averageDurationMs)}ms`,
`Average duration (passed): ${formatNullableDuration(result.averagePassedDurationMs ?? null)}`,
];
if (result.averageTokenUsagePerPassedAttempt) {
lines.push(
`Average tokens (passed): ${formatTokenUsage(result.averageTokenUsagePerPassedAttempt)}`,
);
}
if (result.passedAttempts < result.attemptCount) {
lines.push(
`Average duration (all attempts): ${Math.round(result.averageDurationMs)}ms`,
);
if (result.averageTokenUsagePerAttempt) {
lines.push(
`Average tokens (all attempts): ${formatTokenUsage(result.averageTokenUsagePerAttempt)}`,
);
}
}
const failures = collectFailures(result);
if (failures.length > 0) {
lines.push("Failures:");
@@ -172,6 +175,60 @@ function collectFailures(result: BenchmarkRunResult): string[] {
return failures;
}
function aggregateAttempts(attempts: BenchmarkAttemptResult[]): AttemptAggregate {
const aggregate: AttemptAggregate = {
attemptCount: attempts.length,
durationTotal: 0,
tokenUsageAttemptCount: 0,
tokenUsageTotal: null,
};
for (const attempt of attempts) {
aggregate.durationTotal += attempt.durationMs;
if (!attempt.tokenUsage) {
continue;
}
aggregate.tokenUsageAttemptCount += 1;
aggregate.tokenUsageTotal ??= { prompt: 0, completion: 0, total: 0 };
aggregate.tokenUsageTotal.prompt += attempt.tokenUsage.prompt;
aggregate.tokenUsageTotal.completion += attempt.tokenUsage.completion;
aggregate.tokenUsageTotal.total += attempt.tokenUsage.total;
}
return aggregate;
}
function averageDuration(aggregate: AttemptAggregate): number | null {
return aggregate.attemptCount === 0
? null
: aggregate.durationTotal / aggregate.attemptCount;
}
function averageTokenUsage(
aggregate: AttemptAggregate,
denominator: number,
): BenchmarkTokenUsage | null {
if (denominator === 0 || !aggregate.tokenUsageTotal) {
return null;
}
return {
prompt: aggregate.tokenUsageTotal.prompt / denominator,
completion: aggregate.tokenUsageTotal.completion / denominator,
total: aggregate.tokenUsageTotal.total / denominator,
};
}
function formatNullableDuration(value: number | null): string {
return value === null ? "n/a" : `${Math.round(value)}ms`;
}
function formatTokenUsage(value: BenchmarkTokenUsage): string {
const total = Math.round(value.total);
const prompt = Math.round(value.prompt);
const completion = Math.round(value.completion);
return `${total} total (${prompt} prompt, ${completion} completion)`;
}
function defaultFileName(mode: EvalMode): string {
return `${new Date().toISOString().replaceAll(":", "-")}__${mode}.json`;
}
@@ -252,12 +309,15 @@ function toHistoryRecord(result: BenchmarkRunResult) {
passedAttempts: result.passedAttempts,
passRate: result.passRate,
averageDurationMs: result.averageDurationMs,
averagePassedDurationMs: result.averagePassedDurationMs ?? null,
averageJudgeScore:
judgeScores.length === 0
? null
: judgeScores.reduce((sum, score) => sum + score, 0) /
judgeScores.length,
averageTokenUsagePerAttempt: result.averageTokenUsagePerAttempt ?? null,
averageTokenUsagePerPassedAttempt:
result.averageTokenUsagePerPassedAttempt ?? null,
failedCaseIds: Array.from(
new Set(
result.cases
@@ -268,31 +328,15 @@ function toHistoryRecord(result: BenchmarkRunResult) {
),
),
cases: result.cases.map((caseResult) => {
const attemptCount = caseResult.attempts.length;
const passedAttempts = caseResult.attempts.filter(
(attempt) => attempt.passed,
).length;
const totalDurationMs = caseResult.attempts.reduce(
(sum, attempt) => sum + attempt.durationMs,
0,
const attemptAggregate = aggregateAttempts(caseResult.attempts);
const passedAttemptAggregate = aggregateAttempts(
caseResult.attempts.filter((attempt) => attempt.passed),
);
const attemptCount = attemptAggregate.attemptCount;
const passedAttempts = passedAttemptAggregate.attemptCount;
const judgeScores = caseResult.attempts.flatMap((attempt) =>
typeof attempt.judgeScore === "number" ? [attempt.judgeScore] : [],
);
const totalTokenUsage =
caseResult.attempts.reduce<BenchmarkTokenUsage | null>(
(sum, attempt) => {
if (!attempt.tokenUsage) {
return sum;
}
sum ??= { prompt: 0, completion: 0, total: 0 };
sum.prompt += attempt.tokenUsage.prompt;
sum.completion += attempt.tokenUsage.completion;
sum.total += attempt.tokenUsage.total;
return sum;
},
null,
);
return {
id: caseResult.id,
@@ -300,20 +344,23 @@ function toHistoryRecord(result: BenchmarkRunResult) {
passedAttempts,
passRate: attemptCount === 0 ? 0 : passedAttempts / attemptCount,
averageDurationMs:
attemptCount === 0 ? 0 : totalDurationMs / attemptCount,
attemptCount === 0
? 0
: attemptAggregate.durationTotal / attemptCount,
averagePassedDurationMs: averageDuration(passedAttemptAggregate),
averageJudgeScore:
judgeScores.length === 0
? null
: judgeScores.reduce((sum, score) => sum + score, 0) /
judgeScores.length,
averageTokenUsagePerAttempt:
attemptCount === 0 || !totalTokenUsage
attemptCount === 0
? null
: {
prompt: totalTokenUsage.prompt / attemptCount,
completion: totalTokenUsage.completion / attemptCount,
total: totalTokenUsage.total / attemptCount,
},
: averageTokenUsage(attemptAggregate, attemptCount),
averageTokenUsagePerPassedAttempt: averageTokenUsage(
passedAttemptAggregate,
passedAttempts,
),
};
}),
};
ai_evals/core/types.ts
+15 -1
View File
@@ -110,7 +110,9 @@ export interface AppValidationSpec {
export interface GlobalDraftRequirement {
type: string;
path: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
language?: string;
summaryIncludes?: string[];
@@ -153,6 +155,15 @@ export interface ToolCallArgumentRule {
field: string;
stringStartsWithAnyOf?: string[];
stringMustNotStartWithAnyOf?: string[];
/**
* Case-insensitive "contains", existential over calls: at least one recorded
* call to `tool` must have `field` containing one of these substrings. Other
* calls to the same tool may do anything. Use instead of `stringStartsWithAnyOf`
* (which is universal over calls) when the meaningful token can appear anywhere
* in the value and the model may make additional, unrelated calls to the same
* tool — e.g. SQL where a mutation is mixed with verification SELECTs.
*/
stringIncludesAnyOf?: string[];
}
export interface ToolValidationSpec {
@@ -324,8 +335,11 @@ export interface BenchmarkRunResult {
passedAttempts: number;
passRate: number;
averageDurationMs: number;
averagePassedDurationMs?: number | null;
totalTokenUsage?: BenchmarkTokenUsage | null;
totalPassedTokenUsage?: BenchmarkTokenUsage | null;
averageTokenUsagePerAttempt?: BenchmarkTokenUsage | null;
averageTokenUsagePerPassedAttempt?: BenchmarkTokenUsage | null;
artifactsPath?: string | null;
cases: BenchmarkCaseResult[];
}
ai_evals/core/validators.test.ts
+168
View File
@@ -140,6 +140,111 @@ describe("validateToolExpectations", () => {
details: "tools used: write_script, deploy_workspace_item",
});
});
it("accepts a stringIncludesAnyOf substring regardless of case or position", () => {
const checks = validateToolExpectations({
run: {
success: true,
actual: {},
assistantMessageCount: 1,
toolCallCount: 1,
toolsUsed: ["exec_datatable_sql"],
toolCallDetails: [
{
name: "exec_datatable_sql",
arguments: {
sql: "WITH recent AS (SELECT * FROM orders) SELECT count(*) FROM recent",
},
},
],
skillsInvoked: [],
},
toolExpect: {
requiredToolsUsed: ["exec_datatable_sql"],
toolCallArgs: [
{
tool: "exec_datatable_sql",
field: "sql",
stringIncludesAnyOf: ["select"],
},
],
},
});
expect(checks.every((check) => check.passed)).toBe(true);
});
it("accepts stringIncludesAnyOf when only one of several calls matches", () => {
// Existential: a mutation mixed with verification SELECTs still passes.
const checks = validateToolExpectations({
run: {
success: true,
actual: {},
assistantMessageCount: 1,
toolCallCount: 2,
toolsUsed: ["exec_datatable_sql"],
toolCallDetails: [
{
name: "exec_datatable_sql",
arguments: { sql: "UPDATE orders SET status = 'shipped' WHERE id = 2" },
},
{
name: "exec_datatable_sql",
arguments: { sql: "SELECT * FROM orders WHERE id = 2" },
},
],
skillsInvoked: [],
},
toolExpect: {
toolCallArgs: [
{
tool: "exec_datatable_sql",
field: "sql",
stringIncludesAnyOf: ["insert into", "update"],
},
],
},
});
expect(checks.every((check) => check.passed)).toBe(true);
});
it("rejects stringIncludesAnyOf when no call matches any substring", () => {
const checks = validateToolExpectations({
run: {
success: true,
actual: {},
assistantMessageCount: 1,
toolCallCount: 1,
toolsUsed: ["exec_datatable_sql"],
toolCallDetails: [
{
name: "exec_datatable_sql",
arguments: {
sql: "DROP TABLE orders",
},
},
],
skillsInvoked: [],
},
toolExpect: {
toolCallArgs: [
{
tool: "exec_datatable_sql",
field: "sql",
stringIncludesAnyOf: ["insert into", "update"],
},
],
},
});
expect(checks).toContainEqual({
name: "exec_datatable_sql.sql includes a required substring",
passed: false,
details:
'accepted substrings: insert into, update; values: "DROP TABLE orders"',
});
});
});
describe("validateGlobalState", () => {
@@ -195,6 +300,69 @@ describe("validateGlobalState", () => {
});
});
it("accepts a required script draft without an exact path", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/team_tools/friendly_greeting",
language: "bun",
summary: "Friendly greeting helper",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
validate: {
draftCountExactly: 1,
requiredDrafts: [
{
type: "script",
pathIncludes: ["greeting"],
language: "bun",
summaryIncludes: ["Friendly"],
valueIncludes: ["Hello"],
},
],
},
});
expect(checks.every((check) => check.passed)).toBe(true);
});
it("reports flexible global draft path filters when no draft matches", () => {
const checks = validateGlobalState({
actual: {
drafts: [
{
type: "script",
path: "f/team_tools/friendly_greeting",
language: "bun",
value:
"export async function main(name: string) {\n return `Hello, ${name}!`\n}\n",
isDraft: true,
},
],
},
validate: {
requiredDrafts: [
{
type: "script",
pathIncludes: ["invoice"],
},
],
},
});
expect(checks).toContainEqual({
name: "global includes script draft (path includes invoice)",
passed: false,
details: "drafts: script:f/team_tools/friendly_greeting",
});
});
it("does not require a TypeScript entrypoint for non-TypeScript script drafts", () => {
const checks = validateGlobalState({
actual: {
ai_evals/core/validators.ts
+119 -15
View File
@@ -222,6 +222,25 @@ export function validateToolExpectations(input: {
)
);
}
if (rule.stringIncludesAnyOf && rule.stringIncludesAnyOf.length > 0) {
// Existential: at least one call must contain one of the substrings.
// Other calls to the same tool may do anything — this suits SQL, where a
// model mixes the requested statement (e.g. an UPDATE) with verification
// SELECTs that would otherwise fail an "all calls" check.
const needles = rule.stringIncludesAnyOf.map((needle) => needle.toLowerCase());
const hasMatch = values.some(
(value) =>
typeof value === "string" && needles.some((needle) => value.toLowerCase().includes(needle))
);
checks.push(
check(
`${rule.tool}.${rule.field} includes a required substring`,
hasMatch,
`accepted substrings: ${rule.stringIncludesAnyOf.join(", ")}; values: ${summarizeToolValues(values)}`
)
);
}
}
return checks;
@@ -315,10 +334,11 @@ export function validateGlobalState(input: {
}
for (const required of validate.requiredDrafts ?? []) {
const draft = findGlobalDraft(drafts, required.type, required.path, required.triggerKind);
const requirementLabel = formatGlobalDraftRequirement(required);
const draft = findGlobalDraft(drafts, required);
checks.push(
check(
`global includes ${required.type} draft ${required.path}`,
`global includes ${requirementLabel}`,
Boolean(draft),
summarizeGlobalDrafts(drafts)
)
@@ -330,7 +350,7 @@ export function validateGlobalState(input: {
if (required.language !== undefined) {
checks.push(
check(
`${required.type} draft ${required.path} uses ${required.language}`,
`${requirementLabel} uses ${required.language}`,
draft.language === required.language,
`language=${draft.language ?? "(none)"}`
)
@@ -340,7 +360,7 @@ export function validateGlobalState(input: {
for (const snippet of required.summaryIncludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} summary includes '${snippet}'`,
`${requirementLabel} summary includes '${snippet}'`,
normalizeText(draft.summary ?? "").includes(normalizeText(snippet)),
`summary=${draft.summary ?? ""}`
)
@@ -351,7 +371,7 @@ export function validateGlobalState(input: {
for (const snippet of required.valueIncludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} value includes '${snippet}'`,
`${requirementLabel} value includes '${snippet}'`,
normalizeText(valueText).includes(normalizeText(snippet)),
truncateForDetails(valueText)
)
@@ -361,7 +381,7 @@ export function validateGlobalState(input: {
for (const snippet of required.valueExcludes ?? []) {
checks.push(
check(
`${required.type} draft ${required.path} value excludes '${snippet}'`,
`${requirementLabel} value excludes '${snippet}'`,
!normalizeText(valueText).includes(normalizeText(snippet)),
truncateForDetails(valueText)
)
@@ -373,7 +393,7 @@ export function validateGlobalState(input: {
checks.push(
check(
`global does not include ${forbidden.type} draft ${forbidden.path}`,
!findGlobalDraft(drafts, forbidden.type, forbidden.path, forbidden.triggerKind),
!findGlobalDraft(drafts, forbidden),
summarizeGlobalDrafts(drafts)
)
);
@@ -615,16 +635,100 @@ function summarizeProblems(problems: string[], limit = 5): string | undefined {
function findGlobalDraft(
drafts: GlobalDraft[],
type: string,
path: string,
triggerKind?: string
requirement: {
type: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
summaryIncludes?: string[];
valueIncludes?: string[];
valueExcludes?: string[];
}
): GlobalDraft | undefined {
return drafts.find(
(draft) =>
draft.type === type &&
draft.path === path &&
(triggerKind === undefined || draft.triggerKind === triggerKind)
const candidates = drafts.filter((draft) =>
globalDraftMatchesLocator(draft, requirement)
);
return (
candidates.find((draft) => globalDraftMatchesContent(draft, requirement)) ??
candidates[0]
);
}
function globalDraftMatchesLocator(
draft: GlobalDraft,
requirement: {
type: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
}
): boolean {
return (
draft.type === requirement.type &&
(requirement.path === undefined || draft.path === requirement.path) &&
(requirement.pathStartsWith === undefined ||
draft.path.startsWith(requirement.pathStartsWith)) &&
(requirement.pathIncludes ?? []).every((snippet) =>
normalizeText(draft.path).includes(normalizeText(snippet))
) &&
(requirement.triggerKind === undefined ||
draft.triggerKind === requirement.triggerKind)
);
}
function globalDraftMatchesContent(
draft: GlobalDraft,
requirement: {
summaryIncludes?: string[];
valueIncludes?: string[];
valueExcludes?: string[];
}
): boolean {
const summary = normalizeText(draft.summary ?? "");
const value = normalizeText(stringifyGlobalDraftValue(draft.value));
return (
(requirement.summaryIncludes ?? []).every((snippet) =>
summary.includes(normalizeText(snippet))
) &&
(requirement.valueIncludes ?? []).every((snippet) =>
value.includes(normalizeText(snippet))
) &&
(requirement.valueExcludes ?? []).every(
(snippet) => !value.includes(normalizeText(snippet))
)
);
}
function formatGlobalDraftRequirement(
requirement: {
type: string;
path?: string;
pathIncludes?: string[];
pathStartsWith?: string;
triggerKind?: string;
}
): string {
const typeLabel =
requirement.triggerKind === undefined
? requirement.type
: `${requirement.triggerKind} ${requirement.type}`;
if (requirement.path !== undefined) {
return `${typeLabel} draft ${requirement.path}`;
}
const filters = [
...(requirement.pathStartsWith === undefined
? []
: [`path starts with ${requirement.pathStartsWith}`]),
...(requirement.pathIncludes ?? []).map(
(snippet) => `path includes ${snippet}`
),
];
return filters.length === 0
? `${typeLabel} draft`
: `${typeLabel} draft (${filters.join(", ")})`;
}
function summarizeGlobalDrafts(drafts: GlobalDraft[]): string {
ai_evals/fixtures/frontend/global/initial/current_greeting_live_script.json
+66
View File
@@ -0,0 +1,66 @@
{
"workspace": {
"scripts": [
{
"path": "f/evals/global/format_greeting",
"summary": "Format a deployed greeting",
"description": "Returns a plain greeting for a provided name.",
"language": "bun",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"required": ["name"]
},
"content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n"
},
{
"path": "f/evals/global/format_greeting_archive",
"summary": "Archived greeting formatter",
"description": "Older greeting formatter kept for reference.",
"language": "bun",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"required": ["name"]
},
"content": "export async function main(name: string) {\n return `Hi, ${name}`\n}\n"
}
]
},
"liveEditorDrafts": [
{
"type": "script",
"storagePath": "f/evals/global/current_greeting",
"effectivePath": "f/evals/global/current_greeting",
"value": {
"path": "f/evals/global/current_greeting",
"summary": "Open greeting formatter",
"description": "Formats a greeting in the live editor.",
"language": "bun",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"name": {
"type": "string"
}
},
"required": ["name"]
},
"content": "export async function main(name: string) {\n return `Hello, ${name}`\n}\n",
"is_template": false,
"kind": "script"
}
}
]
}
ai_evals/fixtures/frontend/global/initial/current_invoice_live_flow.json
+118
View File
@@ -0,0 +1,118 @@
{
"workspace": {
"flows": [
{
"path": "f/evals/global/process_invoice",
"summary": "Deployed invoice processor",
"description": "Calculates invoice totals from a subtotal.",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"subtotal": {
"type": "number"
}
},
"required": ["subtotal"]
},
"value": {
"modules": [
{
"id": "calculate_total",
"summary": "Calculate total from subtotal",
"value": {
"type": "rawscript",
"language": "bun",
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
"input_transforms": {
"subtotal": {
"type": "javascript",
"expr": "flow_input.subtotal"
}
}
}
}
]
}
},
{
"path": "f/evals/global/process_refund",
"summary": "Refund processor",
"description": "Calculates refund totals.",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"subtotal": {
"type": "number"
}
},
"required": ["subtotal"]
},
"value": {
"modules": [
{
"id": "calculate_total",
"summary": "Calculate refund total",
"value": {
"type": "rawscript",
"language": "bun",
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
"input_transforms": {
"subtotal": {
"type": "javascript",
"expr": "flow_input.subtotal"
}
}
}
}
]
}
}
]
},
"liveEditorDrafts": [
{
"type": "flow",
"storagePath": "f/evals/global/current_invoice_flow",
"effectivePath": "f/evals/global/current_invoice_flow",
"value": {
"path": "f/evals/global/current_invoice_flow",
"summary": "Open invoice processor",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"subtotal": {
"type": "number"
}
},
"required": ["subtotal"]
},
"value": {
"modules": [
{
"id": "calculate_total",
"summary": "Calculate total from subtotal",
"value": {
"type": "rawscript",
"language": "bun",
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
"input_transforms": {
"subtotal": {
"type": "javascript",
"expr": "flow_input.subtotal"
}
}
}
}
]
},
"edited_by": "",
"edited_at": "",
"archived": false,
"extra_perms": {}
}
}
]
}
ai_evals/fixtures/frontend/global/initial/datatable_orders_seed.json
+39
View File
@@ -0,0 +1,39 @@
{
"workspace": {
"datatables": [
{
"datatable_name": "main",
"schemas": {
"public": {
"orders": {
"columns": {
"id": "int4",
"customer_id": "int4",
"total": "numeric",
"status": "text",
"created_at": "timestamptz"
},
"rows": [
{ "id": 1, "customer_id": 1, "total": 42.5, "status": "shipped", "created_at": "2026-05-01T10:00:00Z" },
{ "id": 2, "customer_id": 2, "total": 19.99, "status": "pending", "created_at": "2026-05-02T11:30:00Z" },
{ "id": 3, "customer_id": 1, "total": 88, "status": "shipped", "created_at": "2026-05-03T09:15:00Z" }
]
},
"customers": {
"columns": {
"id": "int4",
"name": "text",
"email": "text",
"tier": "text"
},
"rows": [
{ "id": 1, "name": "Alice", "email": "alice@example.com", "tier": "gold" },
{ "id": 2, "name": "Bob", "email": "bob@example.com", "tier": "silver" }
]
}
}
}
}
]
}
}
ai_evals/fixtures/frontend/global/initial/process_invoice_flow.json
+40
View File
@@ -0,0 +1,40 @@
{
"workspace": {
"flows": [
{
"path": "f/evals/global/process_invoice",
"summary": "Process an invoice subtotal",
"description": "Calculates invoice totals from a subtotal.",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"subtotal": {
"type": "number"
}
},
"required": ["subtotal"]
},
"value": {
"modules": [
{
"id": "calculate_total",
"summary": "Calculate total from subtotal",
"value": {
"type": "rawscript",
"language": "bun",
"content": "export async function main(subtotal: number) {\n return { subtotal, total: subtotal }\n}\n",
"input_transforms": {
"subtotal": {
"type": "javascript",
"expr": "flow_input.subtotal"
}
}
}
}
]
}
}
]
}
}
ai_evals/fixtures/frontend/global/initial/report_digest_script.json
+23
View File
@@ -0,0 +1,23 @@
{
"workspace": {
"scripts": [
{
"path": "f/evals/global/send_report_digest",
"summary": "Build and send the eval report digest",
"description": "Returns a dry-run summary for eval report digest notifications.",
"language": "bun",
"schema": {
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"dry_run": {
"type": "boolean"
}
},
"required": ["dry_run"]
},
"content": "export async function main(dry_run: boolean) {\n return { dry_run, sent: !dry_run, message: dry_run ? 'Preview digest' : 'Digest sent' }\n}\n"
}
]
}
}
ai_evals/modes/global.ts
+7 -1
View File
@@ -1,5 +1,8 @@
import { readFile } from "node:fs/promises";
import { runGlobalEval } from "../adapters/frontend/core/global/globalEvalRunner";
import {
runGlobalEval,
type GlobalLiveEditorDraftFixture,
} from "../adapters/frontend/core/global/globalEvalRunner";
import type { BenchmarkWorkspaceRunnables } from "../adapters/frontend/mockBackend";
import type { FrontendEvalModelConfig } from "../core/models";
import type { BenchmarkArtifactFile, GlobalValidationSpec, ModeRunner } from "../core/types";
@@ -9,6 +12,7 @@ import { getFrontendApiKey } from "./frontendCommon";
export interface GlobalInitialFixture {
workspace?: BenchmarkWorkspaceRunnables;
liveEditorDrafts?: GlobalLiveEditorDraftFixture[];
}
export function createGlobalModeRunner(
@@ -31,6 +35,7 @@ export function createGlobalModeRunner(
getFrontendApiKey(modelConfig.provider),
{
workspaceFixtures: initial?.workspace,
liveEditorDrafts: initial?.liveEditorDrafts,
maxIterations: context.evalCase?.runtime?.maxTurns,
provider: modelConfig.provider,
model: modelConfig.model,
@@ -73,6 +78,7 @@ async function loadGlobalInitialFixture(path: string): Promise<GlobalInitialFixt
const parsed = JSON.parse(await readFile(path, "utf8")) as GlobalInitialFixture;
return {
workspace: parsed.workspace ?? {},
liveEditorDrafts: parsed.liveEditorDrafts ?? [],
};
}
backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json
Generated
+5 -5
View File
@@ -46,11 +46,11 @@
]
},
"nullable": [
true,
true,
true,
true,
true,
false,
false,
false,
false,
false,
true,
true
]
backend/.sqlx/query-0cca68f11329cd41ab9372297b715af008ea7db408f49cf525656d6224092429.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT operator, is_admin FROM usr WHERE email = $1 AND is_service_account IS true LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "operator",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "is_admin",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "0cca68f11329cd41ab9372297b715af008ea7db408f49cf525656d6224092429"
}
backend/.sqlx/query-19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT id AS \"id!\" FROM chain",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "19513c4158267cc7fe10d999ad571052c112e6bbb3cf834f16176cbb7e1ac319"
}
backend/.sqlx/query-1d0341bd8de94ab8d34a4bb1bb2005305fb3b29751ec987bece183444e89e7d1.json
Generated
+100
View File
@@ -0,0 +1,100 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email, login_type::TEXT, super_admin, devops, verified, name, company, username, NULL::bool as operator_only, NULL::bool as is_workspace_admin, first_time_user, role_source, disabled, NULL::text as workspace_id FROM password WHERE email = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "devops",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "verified",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "is_workspace_admin",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "first_time_user",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "role_source",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "disabled",
"type_info": "Bool"
},
{
"ordinal": 13,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null,
false,
false,
false,
true,
true,
true,
null,
null,
false,
false,
false,
null
]
},
"hash": "1d0341bd8de94ab8d34a4bb1bb2005305fb3b29751ec987bece183444e89e7d1"
}
backend/.sqlx/query-6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a.json
Generated
+38
View File
@@ -0,0 +1,38 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH capped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n UNION ALL\n SELECT timestamp, operation, resource, parameters\n FROM audit\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.capped'\n ORDER BY timestamp DESC\n LIMIT 200\n ), uncapped AS (\n SELECT timestamp, operation, resource, parameters\n FROM audit_partitioned\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n UNION ALL\n SELECT timestamp, operation, resource, parameters\n FROM audit\n WHERE workspace_id = 'admins'\n AND operation = 'workspace_fairness.uncapped'\n ORDER BY timestamp DESC\n LIMIT 200\n )\n SELECT timestamp AS \"timestamp!\",\n operation::text AS \"operation!\",\n resource AS workspace_id,\n parameters\n FROM (SELECT * FROM capped UNION ALL SELECT * FROM uncapped) e\n ORDER BY timestamp DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "timestamp!",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "operation!",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "parameters",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null,
null
]
},
"hash": "6fcdb09cdd7fedd7e54fdc0e49203f453fc1b85272fe212c0e1bf0ebd28bf58a"
}
backend/.sqlx/query-7f832370916794ab0e5645053688c24678f1519d49ee7263a86dba71d45b8e8c.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO token\n (token_hash, token_prefix, token, email, label, expiration, super_admin, scopes, read_only)\n VALUES ($1, $2, $3, $4, $5, now() + ($6 || ' seconds')::interval, $7, $8, $9)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Text",
"Bool",
"TextArray",
"Bool"
]
},
"nullable": []
},
"hash": "7f832370916794ab0e5645053688c24678f1519d49ee7263a86dba71d45b8e8c"
}
backend/.sqlx/query-815d96aea4681490582b08630a30a168cc1191acaab96bed6a016c437059c2cd.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT ws.workspace_id AS \"workspace_id!\", entry->'catalog'->>'resource_path' AS dbname\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(\n CASE WHEN jsonb_typeof(ws.ducklake->'ducklakes') = 'object'\n THEN ws.ducklake->'ducklakes'\n ELSE '{}'::jsonb END\n ) AS dl(k, entry)\n WHERE entry->'catalog'->>'resource_type' = 'instance'\n AND entry->'catalog'->>'resource_path' IS NOT NULL\n UNION ALL\n SELECT ws.workspace_id AS \"workspace_id!\", entry->'database'->>'resource_path' AS dbname\n FROM workspace_settings ws\n CROSS JOIN LATERAL jsonb_each(\n CASE WHEN jsonb_typeof(ws.datatable->'datatables') = 'object'\n THEN ws.datatable->'datatables'\n ELSE '{}'::jsonb END\n ) AS dt(k, entry)\n WHERE entry->'database'->>'resource_type' = 'instance'\n AND entry->'database'->>'resource_path' IS NOT NULL\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "dbname",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "815d96aea4681490582b08630a30a168cc1191acaab96bed6a016c437059c2cd"
}
backend/.sqlx/query-8597cd40f80e69edbf1bc7d7402baca32e33e871be454acb5175c11361fe1b0a.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM background_task_state\n WHERE name LIKE $1\n AND updated_at < NOW() - INTERVAL '7 days'",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "8597cd40f80e69edbf1bc7d7402baca32e33e871be454acb5175c11361fe1b0a"
}
backend/.sqlx/query-8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000.json
Generated
+101
View File
@@ -0,0 +1,101 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users AS (SELECT distinct username as email FROM (SELECT username, timestamp, operation FROM audit_partitioned UNION ALL SELECT username, timestamp, operation FROM audit) AS a WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh')),\n authors as (SELECT distinct email FROM usr WHERE usr.operator IS false)\n SELECT email as \"email!\", (email NOT IN (SELECT email FROM authors)) as operator_only, NULL::bool as is_workspace_admin, login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id\n FROM password\n WHERE email IN (SELECT email FROM active_users)\n UNION ALL\n SELECT email as \"email!\", operator as operator_only, is_admin as is_workspace_admin, 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "is_workspace_admin",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "verified!",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "super_admin!",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "devops!",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 10,
"name": "first_time_user!",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "role_source!",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "disabled!",
"type_info": "Bool"
},
{
"ordinal": 13,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
]
},
"hash": "8aae160c589adf02e20b7e6ba860b66fb08f156776b96b7f3701aed330a7d000"
}
backend/.sqlx/query-8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = $1 AND workspace_id = $2 AND tag = ANY($3))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text",
"TextArray"
]
},
"nullable": [
null
]
},
"hash": "8e8933fc6648a88dc35cd81559a31d10678d6c68fc920c876914e71324d5e460"
}
backend/.sqlx/query-afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE app SET versions = array_append(versions, $1::bigint) WHERE path = $2 AND workspace_id = $3 AND versions[array_upper(versions, 1)] = $1::bigint",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "afb0762c88d9232b79090f2e5966e78437a5e4d3b5e2341ec5f7725a28870270"
}
backend/.sqlx/query-b876f26ce90e30c3510eacddb03d9dd26fac05d18183f2d623ac91ea3876dd5c.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms, account)\n VALUES ($1, $2, $3, true, '', '{}'::jsonb, NULL)\n ON CONFLICT (workspace_id, path) DO UPDATE SET value = EXCLUDED.value\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "b876f26ce90e30c3510eacddb03d9dd26fac05d18183f2d623ac91ea3876dd5c"
}
backend/.sqlx/query-bce66e3f3fecda6c226556f2f3cff27702b8fc8d2850fdb262b9a2a77b468646.json
Generated
+101
View File
@@ -0,0 +1,101 @@
{
"db_name": "PostgreSQL",
"query": "SELECT email as \"email!\", login_type::text, verified as \"verified!\", super_admin as \"super_admin!\", devops as \"devops!\", name, company, username, NULL::bool as operator_only, NULL::bool as is_workspace_admin, first_time_user as \"first_time_user!\", role_source as \"role_source!\", disabled as \"disabled!\", NULL::text as workspace_id FROM password\n UNION ALL\n SELECT email as \"email!\", 'service_account'::text as login_type, true as \"verified!\", false as \"super_admin!\", false as \"devops!\", NULL::text as name, NULL::text as company, username, operator as operator_only, is_admin as is_workspace_admin, false as \"first_time_user!\", 'service_account'::text as \"role_source!\", disabled as \"disabled!\", workspace_id\n FROM usr\n WHERE is_service_account IS true\n ORDER BY \"super_admin!\" DESC, \"devops!\" DESC, \"email!\"\n LIMIT $1 OFFSET $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "login_type",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "verified!",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "super_admin!",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "devops!",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "company",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "operator_only",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "is_workspace_admin",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "first_time_user!",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "role_source!",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "disabled!",
"type_info": "Bool"
},
{
"ordinal": 13,
"name": "workspace_id",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8",
"Int8"
]
},
"nullable": [
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null,
null
]
},
"hash": "bce66e3f3fecda6c226556f2f3cff27702b8fc8d2850fdb262b9a2a77b468646"
}
backend/.sqlx/query-bf601a919de299e44e6e418b2e711e24909fc206b7fd48541483e6316fe002d5.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT j.id, j.args\n FROM v2_job j\n JOIN v2_job_queue q ON j.id = q.id\n WHERE j.runnable_path = $1\n AND j.kind = 'deploymentcallback'\n AND j.workspace_id = 'test-workspace'\n ORDER BY j.created_at DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "args",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
true
]
},
"hash": "bf601a919de299e44e6e418b2e711e24909fc206b7fd48541483e6316fe002d5"
}
backend/.sqlx/query-c533691be8136c5ed6835c2fcdb016c257a5e4ee271d221fa961c24bd119d98e.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT permissioned_as_email FROM v2_job WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "permissioned_as_email",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "c533691be8136c5ed6835c2fcdb016c257a5e4ee271d221fa961c24bd119d98e"
}
backend/.sqlx/query-ca5bb402834502432f3d7260fdd5b9fb568a4c77e2a91f55575a93461d5a7f50.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM v2_job WHERE id = ANY($1) AND workspace_id = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": [
null
]
},
"hash": "ca5bb402834502432f3d7260fdd5b9fb568a4c77e2a91f55575a93461d5a7f50"
}
backend/.sqlx/query-cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037.json
Generated
+60
View File
@@ -0,0 +1,60 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, script_lang AS \"script_lang: ScriptLang\" FROM v2_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "script_lang: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java",
"duckdb",
"ruby",
"rlang"
]
}
}
}
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false,
true
]
},
"hash": "cce5e3e639faed8e42574730cc66f0322a83c01cc465742f54a21f8fe5f4f037"
}
backend/.sqlx/query-dd540bcb206d931eb19aa85059b6a64554a9a29ebc6064718072470e45a0ae28.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO usr\n (workspace_id, email, username, is_admin, operator, is_service_account)\n VALUES ($1, $2, $3, $4, $5, true)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Bool"
]
},
"nullable": []
},
"hash": "dd540bcb206d931eb19aa85059b6a64554a9a29ebc6064718072470e45a0ae28"
}
backend/.sqlx/query-e88e1009f5359e205a523a32e4e8e72605971a3f417cb2b032c4d43652aca056.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "SELECT hash, content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "hash",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "content",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false
]
},
"hash": "e88e1009f5359e205a523a32e4e8e72605971a3f417cb2b032c4d43652aca056"
}
backend/.sqlx/query-f699cc3644aeb35a0588bbb3a6bf2dc0746d9f3a1bea104123188fe2921bc886.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE workspace_runnable_dependencies SET app_path = REGEXP_REPLACE(app_path,'u/' || $2 || '/(.*)','u/' || $1 || '/\\1') WHERE app_path LIKE ('u/' || $2 || '/%') AND workspace_id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "f699cc3644aeb35a0588bbb3a6bf2dc0746d9f3a1bea104123188fe2921bc886"
}
backend/Cargo.lock
Generated
+389 -389
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+3 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.708.0"
version = "1.719.0"
authors.workspace = true
edition.workspace = true
@@ -87,7 +87,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.708.0"
version = "1.719.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -251,6 +251,7 @@ windmill-object-store.workspace = true
windmill-git-sync.workspace = true
windmill-api = { workspace = true, default-features = false }
windmill-api-agent-workers = { workspace = true, optional = true }
windmill-api-scripts.workspace = true
windmill-api-settings.workspace = true
windmill-worker.workspace = true
windmill-indexer = { workspace = true, optional = true }
backend/THREAT_MODEL.md
+172
View File
@@ -0,0 +1,172 @@
# Threat Model: Windmill Backend
## 1. System context
Windmill is an open-source (AGPLv3) developer platform for internal tools,
workflows, background jobs, API integrations, and UIs — a self-hostable
alternative to Retool / Pipedream / Airplane. The backend is a Rust workspace
(~60 crates: `windmill-api`, `windmill-worker`, `windmill-queue`,
`windmill-common`, a family of `windmill-trigger-*` crates, `windmill-mcp`,
`windmill-sandbox`, etc.) fronting a PostgreSQL database. A Svelte 5 frontend
(not in scope here, but referenced where stored-XSS threats originate) is
served by the same instance. The product ships in a Community Edition (CE,
public Docker images) and an Enterprise Edition (EE, `*_ee.rs` files gated by
`enterprise`/`private`/`license` cargo features).
The defining characteristic for threat modeling is that **Windmill executes
arbitrary user-supplied code** (Python, TypeScript via Bun/Deno, Go, Bash,
SQL, GraphQL, PowerShell, Rust, …) on its workers, and **stores the
credentials to every system its users connect to** (databases, cloud
accounts, SaaS APIs, OAuth tokens). It is therefore simultaneously an
arbitrary-code-execution engine and a credential vault — compromising one
instance can pivot into an organization's entire connected estate. Crucially,
the owner confirms `nsjail` is **off by default everywhere** (`ENABLE_NSJAIL`
is opt-in) and network isolation (`clone_newnet`) is separately gated: the
*only* job isolation present in a default install is PID-namespace `unshare`.
Filesystem and outbound-network isolation are therefore absent unless an
operator deliberately enables them, which makes "weak-by-default isolation" a
more accurate frame than "sandbox escape" for typical deployments. Cross-tenant
separation is enforced in software via workspace IDs, token scopes, folder
ACLs, and Postgres row-level security; on the managed offering, sensitive
customers can opt into dedicated DB / worker / namespace infrastructure, but
the shared tier relies entirely on that software boundary. Administrators are
strongly encouraged to use nsjail sandboxing and are reminded that if they don't,
their security model is that they trust their developers that write code ran on windmill
to not do anything TOO malicious on the workers. When the default
database secret backend is used, only per-workspace secret *variables* are
encrypted at rest — instance-level `global_settings` (OAuth client secrets,
SMTP, object-store keys, license) are stored plaintext, so a database read
yields the instance-wide credential set. Internet-facing instances are
typically exposed directly with no built-in rate limiting or WAF.
It is deployed self-hosted (Docker Compose, Kubernetes/Helm, bare metal), on
cloud providers, and as a Windmill-Labs-managed multi-tenant service. The API
server is internet-facing in most deployments; workers pull jobs from the
Postgres queue. The large public attack surface (a sprawling authenticated
HTTP API, unauthenticated public-app and webhook/trigger endpoints, outbound
HTTP from user code and proxies) combined with the high-value assets makes
authorization-enforcement bugs, SSRF, SQL injection, and sandbox escape the
dominant risk categories — a pattern strongly confirmed by the project's
published advisory history (73 GHSA advisories, several rated 9.9 critical).
## 2. Assets
| asset | description | sensitivity |
|---|---|---|
| Workspace encryption keys | Per-workspace key (`workspace_key`) used to encrypt secret variables (MagicCrypt256); decrypts all secrets in the workspace | critical |
| Secret variables | User secrets stored encrypted in `variable` (is_secret) | critical |
| Resource credentials | DB passwords, cloud creds, API keys, connection strings in `resource` JSONB | critical |
| OAuth / external-account tokens | Refresh/access tokens in `account`, MCP OAuth tables | critical |
| User password hashes | Argon2 hashes in `password` table | critical |
| API tokens & session cookies | Bearer tokens / cookies in `token`; superadmin & scoped tokens | critical |
| Instance global settings | License key, JWT secret, SUPERADMIN_SECRET, SMTP, object-store + secret-backend (Vault/KMS/SM) creds in `global_settings` | critical |
| Worker host & process integrity | The host that runs untrusted user code | critical |
| Cross-tenant / cross-workspace isolation | The software boundary separating workspaces, folders, and tenants | critical |
| Downstream connected systems | Windmill is a credential vault: stored creds reach external DBs, cloud accounts, SaaS | critical |
| Script / flow / app source | Customer IP & business logic in `script`, `flow`, `app`, `raw_app` | high |
| Job arguments, results & logs | `queue`/`completed_job` args+result, `job_logs`; routinely contain secrets | high |
| Object store / S3 data | Files uploaded/produced by jobs | high |
| Audit logs | `audit`/`audit_partitioned` action trail | high |
| Service availability | API server + worker fleet uptime | high |
| PII | User emails, group membership | medium |
## 3. Entry points & trust boundaries
| entry_point | description | trust_boundary | reachable_assets |
|---|---|---|---|
| EP1 Authenticated job-execution API | `jobs/run/preview`, `run/h/{hash}`, `run_flow/run_script` — runs user code on workers | authenticated user → arbitrary code on worker | Worker host, downstream systems, isolation, job args/results/logs |
| EP2 Unauthenticated public endpoints | `apps_u/*`, `jobs_u/getupdate*`, `scripts_u`, `settings_u`, `resources_u` (`public_app_layer.rs`) | unauth HTTP → app logic & job data | Job results, scripts, secrets, PII |
| EP3 HTTP-trigger & webhook ingestion | `/api/r/*`, GCP/Azure push, Slack callback, `capture_u/*` | untrusted webhook → job queue | Job execution integrity, worker host |
| EP4 Message-queue / native triggers | kafka, postgres, mqtt, websocket, nats, sqs, email triggers | external broker/message → job queue | Job execution integrity, availability |
| EP5 HTTP API authorization layer | Token/scope/RLS/folder-ACL enforcement across all workspaced routes (`windmill-api-auth`) | scoped token / low-priv user → other users' & workspaces' data | Scripts, job data, secrets, isolation |
| EP6 AI proxy & MCP endpoints | `ai/proxy/*`, `mcp` — resolve `$var:`/resources, proxy to LLM APIs, `X-Resource-Path` | authenticated user → outbound HTTP + secret resolution | Secrets, resource creds, internal network, downstream |
| EP7 Outbound HTTP from executors/resources | GraphQL/HTTP/Postgres executors, webhook delivery, `test_object_storage_config`, git clone, npm tarball fetch | user-controlled URL → server-side request | Cloud metadata, internal network, downstream creds |
| EP8 SQL query builders & contextual-var substitution | App DB query builder (`whereClause`/`tags`), Postgres-trigger `where_clause`, `%%WM_*%%` interpolation, `WM_INTERNAL_DB` | user input → raw SQL | Database, connected DBs |
| EP9 Worker sandbox | nsjail / unshare / dind / rootless podman isolating user code | user code → host & cross-tenant filesystem/network | Worker host, isolation, downstream |
| EP10 Worker code generation / wrappers | Entrypoint override, env-var names, workspace env interpolated into generated wrapper code | user-controlled identifier → executable code | Worker host, isolation |
| EP11 OAuth / OIDC / SAML / MCP-OAuth / logout | Login callbacks, MCP OAuth client registration, logout `rd` redirect | untrusted IdP / redirect input → session | Session tokens, accounts |
| EP12 Stored-content rendering | App builder HTML component, markdown, S3 download response headers | stored user content → admin browser (same origin) | Admin session, account takeover |
| EP13 Log/file reading & export endpoints | `service_logs`, `jobs_u/getupdate` log file read (symlinks), workspace/tarball export | authed/unauth request → arbitrary file or admin-only config | Arbitrary files, global settings |
| EP14 Secret-value & resource-value caches | In-memory caches in `windmill-store` keyed (historically un-keyed) by path | cache lookup crossing identity/folder boundary | Secret variables, resource creds |
| EP15 Deployment & runtime config | docker-compose defaults: dind, debugger (`REQUIRE_SIGNED_DEBUG_REQUESTS=false`), CORS `Any`, default admin/`changeme`, exposed Postgres, `SUPERADMIN_SECRET`, `ENABLE_NSJAIL=false`, privileged containers | operator/infra default → full instance | All assets |
| EP16 Supply chain | Cached hub scripts, GitHub workflow actions, vendored deps, Docker base image | build/update-time input → host & build integrity | Worker host, build integrity |
| EP17 Token lifecycle | Token create/rescope/refresh, script-issued JWTs | scoped caller → broader privilege | Tokens, accounts, isolation |
## 4. Threats
| id | threat | actor | surface | asset | impact | likelihood | status | controls | evidence |
|---|---|---|---|---|---|---|---|---|---|
| T1 | SQL injection in app/internal query builders and trigger clauses compromises the metadata DB and connected databases | remote_auth | EP8 | Database, downstream connected systems | critical | almost_certain | partially_mitigated | sqlx parameterized queries elsewhere; query-builder safety reviews | GHSA-225c-j3xq-g6x6, GHSA-78p7-jc72-gv66, GHSA-hvc7-f67h-jx3g, GHSA-wrrg-f89m-f84q, GHSA-79vf-3qwm-2w64, GHSA-55p6-fxj4-v983, GHSA-5g4v-49rj-r52r, GHSA-x6cq-7xr8-53x3, 2cf4bb180b |
| T2 | Server-side request forgery via proxies/executors reaches cloud metadata, internal network, and downstream credentials | remote_auth | EP6, EP7 | Cloud metadata, internal network, downstream connected systems, resource creds | critical | almost_certain | partially_mitigated | SSRF URL validation + redirect-following disabled added piecemeal; outbound network isolation (`clone_newnet`) is opt-in and off by default | GHSA-3ggp-h37f-5qfw, GHSA-98qq-g8rh-xhff, GHSA-hfw8-27mx-63jm, GHSA-3r59-qvvc-774j, GHSA-4pj9-w5jc-g8w7, GHSA-8hh3-jf25-78j5, GHSA-3pjm-4w7f-3r2w, GHSA-f44c-x9hq-h68r, GHSA-j4h4-f8fj-3m3c, 4b06881918, 96a8eb63d4, dbd3942ef3 |
| T3 | Broken authorization / IDOR lets a scoped token or low-privilege member read scripts, job data, and secrets across folders and workspaces | remote_auth | EP5, EP2, EP1 | Scripts, job data, secrets, isolation | critical | almost_certain | partially_mitigated | RLS, token scopes, folder ACLs, view-token HMAC (added incrementally); on managed, sensitive tenants can opt into dedicated DB/worker/namespace, but the shared tier IS the software boundary | GHSA-qfg7-x243-5hg4, GHSA-8x8x-88qc-qp4r, GHSA-2ppx-66jv-wpw5, GHSA-x3x7-g97v-mp59, GHSA-j276-g4h8-g6h5, GHSA-8mv7-hmrg-96xv, GHSA-x2wf-f962-7frq, GHSA-qc7c-gcw6-h4xp, GHSA-vxc5-w28p-m9xw, GHSA-2g34-wfvr-5qqj, GHSA-w7p6-wpxm-pp66, 7edf3f0212, 89a7a37776, ab11c7747a, 664edcdfb7 |
| T4 | Remote code execution by injecting attacker-controlled identifiers into generated worker wrappers | remote_auth | EP10 | Worker host, isolation, downstream | critical | likely | partially_mitigated | entrypoint/env-var-name validation added | GHSA-wxjq-w5pj-jqhx, GHSA-5f5q-2vg2-r2x4, GHSA-8q8j-mm3g-5c2q (CVE-2026-33881), bf93657fee, bd05bcadde, 22ec4da5f0 |
| T5 | Worker compromise & cross-tenant access via weak-by-default isolation (nsjail off by default → user code runs with only PID-ns `unshare`); sandbox escape where nsjail/dind/podman is enabled | remote_auth | EP9, EP15 | Worker host, isolation, downstream | critical | likely | unmitigated | nsjail off by default everywhere (`DISABLE_NSJAIL=true`); shipped compose gives PID-ns `unshare` only (`FAVOR_UNSHARE_PID=true`), bare installs get no isolation. Where nsjail enabled: read-only remounts, jail-tmp refusal, podman socket gating | GHSA-6qr8-xhg4-453q, GHSA-3vpp-vf62-wqp6, f8467f38c8, df5aec0f5d, f1b6746e0e |
| T6 | Disclosure of secrets, resource credentials, and workspace encryption keys across the authorization boundary (AI proxy, MCP, caches, export); database read additionally yields plaintext instance-level `global_settings` secrets | remote_auth | EP6, EP14, EP13 | Secret variables, encryption keys, resource creds, global settings | critical | likely | partially_mitigated | RLS on `$var:`, cache scoping by caller, admin checks on export; per-workspace secret *variables* encrypted at rest, but `global_settings` is plaintext under the default DB secret backend | GHSA-jwg4-v3cj-rvfm, GHSA-8m2p-2crh-9h3w, GHSA-6635-6fch-v8px, GHSA-437f-725p-7w84, GHSA-f27g-j463-q85w (CVE-2026-26964), GHSA-j679-v6vj-jfxc, GHSA-6vrr-fq33-qpfp, 0ba128afe7, 7836a4e733, ff8e39c69b |
| T7 | Full instance compromise from insecure deployment defaults (dind control, default admin/`changeme`, exposed Postgres, publicly readable SUPERADMIN_SECRET) | remote_unauth | EP15 | All assets | critical | likely | partially_mitigated | first-time-setup warning on default admin; docs recommend hardening | GHSA-3vpp-vf62-wqp6, GHSA-24fr-44f8-fqwg (CVE-2026-29059), GHSA-6q36-5p3h-766j |
| T8 | Unauthenticated RCE via the Debugger WebSocket in the default `windmill_extra` configuration | remote_unauth | EP15 | Worker host, all assets | critical | possible | unmitigated | `REQUIRE_SIGNED_DEBUG_REQUESTS` exists but defaults to false | GHSA-725h-99vx-9xr4 |
| T9 | Supply-chain compromise via cached hub scripts, GitHub workflow command injection, or vulnerable base-image deps | supply_chain | EP16 | Worker host, build integrity | critical | possible | partially_mitigated | hub-script re-pin to patched versions; HUB_BASE_URL override | GHSA-w2m9-q5f7-3gpq, edf340c4d4, GHSA-8rq7-w7g6-8wvr, GHSA-vch9-39v5-4wg7 (CVE-2024-37371) |
| T10 | Unauthenticated disclosure of job results, args, logs, and admin config via missing-authz public endpoints | remote_unauth | EP2, EP13 | Job results/args/logs, global settings, scripts | high | likely | partially_mitigated | anonymous-job checks, log-endpoint authz hardening | GHSA-qfg7-x243-5hg4, GHSA-v448-fmm4-52fp, 108a88a180, bb90f4ce83 |
| T11 | Stored XSS leading to admin/account takeover via app HTML component, markdown, or S3 download content-type | remote_auth | EP12 | Admin session, accounts | high | likely | partially_mitigated | DOMPurify markdown sanitization, `X-Content-Type-Options: nosniff` + CSP sandbox on downloads | GHSA-9c5c-hh3c-r9mc, GHSA-qxj7-hpx3-r892, GHSA-cf2x-rg8c-v63v, bb78b1c06d, 625b67dff0 |
| T12 | Webhook authentication bypass / signature replay forges trigger invocations and approvals | remote_unauth | EP3 | Job execution integrity, approvals | high | likely | partially_mitigated | HMAC verification on some triggers; signing-oracle fix | GHSA-jw8c-h45c-xpjw, GHSA-hh9x-rcf8-xjr2, GHSA-q9g3-q6fj-hc2x, GHSA-8jc4-wj2p-2vmp, ab2a15b2a8 |
| T13 | Path traversal / arbitrary file read via log-reading and MCP path endpoints (incl. symlink following) | remote_auth | EP13 | Arbitrary files on server, global settings | high | likely | partially_mitigated | traversal checks + no-symlink-follow added | GHSA-4hrf-mgvv-xp9x, bb90f4ce83, df451aa64f, ad5ec293b5, 5f2d3e6812 |
| T14 | Privilege escalation via token rescope/refresh, script-issued JWTs, or operator-permission gaps | remote_auth | EP17, EP5 | Tokens, isolation, accounts | high | likely | partially_mitigated | monotonic-privilege enforcement on token lifecycle; SECURITY DEFINER triggers | GHSA-p62p-67xp-v775, GHSA-vv9w-wx3c-q3x2, 2ddf93de96, 865ab70c89, 33fb08cf3d |
| T15 | Credential leakage via worker `/proc` environment and unmasked secrets in job logs | remote_auth | EP9, EP1 | DB creds, secrets, downstream | high | likely | partially_mitigated | Aho-Corasick secret masking in logs | GHSA-pmp9-9924-f9cx, 0885d8c986 |
| T16 | Denial of service via resource exhaustion: unbounded uploads, runaway jobs, queue flooding, or trigger-message storms | remote_auth | EP1, EP3, EP4 | Service availability, worker fleet | high | likely | risk_accepted | Per-job rlimits/timeouts exist; instance-wide DoS by an authenticated tenant is largely accepted on shared self-host (operator's job to add global quotas). Hard requirement only for managed multi-tenant | |
| T17 | Account/credential theft via unauthenticated MCP-OAuth client registration and open redirect on logout | remote_unauth | EP11 | Accounts, session tokens | high | possible | partially_mitigated | redirect-URI handling / registration hardening | GHSA-q9xg-f2v2-695g, GHSA-53xj-pvqf-wpm9, GHSA-rr8j-ffc4-pf7h, GHSA-6c5w-777m-8rv5 |
| T18 | Account takeover via missing rate limiting / brute force on auth endpoints | remote_unauth | EP11 | Accounts | medium | likely | unmitigated | none built-in; owner confirms instances are typically exposed directly with no app-level rate limiting or WAF | GHSA-cmv6-m7wc-c87p |
| T19 | Enterprise license bypass and account impersonation | remote_auth | EP5 | Global settings, accounts | medium | possible | unmitigated | license validation gated by `license` feature | GHSA-48j5-p323-4mpx, GHSA-pv35-65rq-w29h, GHSA-2qx7-634r-qj6r |
| T20 | Trigger spoofing: an actor with broker/queue access injects messages that execute jobs without app-level auth | adjacent_network | EP4 | Job execution integrity, downstream | medium | possible | risk_accepted | Owner confirms trust is delegated to broker ACLs by design; no app-level message authenticity check. Anyone able to publish to a subscribed topic/queue can cause job execution | |
| T21 | Data-in-transit interception/tampering from TLS-disabled defaults (DB `sslmode=disable`, HTTP-only Caddy) | adjacent_network | EP15 | DB creds, secrets, session tokens | medium | possible | unmitigated | docs recommend TLS; not default | |
| T22 | Repudiation / incident blind spots from gaps in audit coverage of sensitive actions | remote_auth | EP5 | Audit logs | medium | possible | partially_mitigated | `windmill-audit` records many actions | |
## 5. Deprioritized
| threat | reason |
|---|---|
| Physical access to the host / cold-boot key extraction | Out of scope; deployment-environment responsibility, not addressable in this codebase |
| Memory-safety RCE in the Rust backend itself | Rust's safety model makes this rare; no evidence in history. Note: `unsafe` FFI (duckdb) is a narrow exception folded into supply-chain/T9 |
| Client-side-only nuisance bugs (CSS, layout) with no security impact | No asset compromised |
| Insider with legitimate superadmin / DB-root access | Trusted role; mitigations are operational (least privilege, audit), not technical controls in scope |
| Spoofing of a fully-trusted upstream IdP that has itself been compromised | Out of model; Windmill trusts the configured IdP by design |
| Instance-wide DoS by an authenticated tenant on shared self-host (T16) | Risk accepted (owner): per-job rlimits/timeouts are in place; global concurrency/queue quotas are the operator's responsibility on self-host. Remains a hard requirement for the managed multi-tenant fleet |
| Job execution triggered by an actor with legitimate broker/queue publish access (T20) | Risk accepted (owner): trigger authenticity is delegated to broker ACLs by design; consuming from a configured source and acting on its messages is the intended behavior |
## 6. Open questions
Facts that drove the score changes above. Two were confirmed in code during
the interview (`[Code-verified]`); the rest remain `[Owner-states]` pending a
check.
- [Code-verified] nsjail is off by default in every configuration: `DISABLE_NSJAIL` defaults to `true` (`windmill-worker/src/worker.rs:346`), and `is_sandboxing_enabled()` requires `DISABLE_NSJAIL=false` or the `job_isolation` global setting = `nsjail_sandboxing` (`worker.rs:890`). PID-ns `unshare` is also off at the code level (`is_unshare_enabled()`, `worker.rs:903`); the shipped `docker-compose.yml` sets `FAVOR_UNSHARE_PID=true` (line 91), so the official compose gives PID-ns unshare only, nsjail off — a bare install gets no isolation at all. No separate `clone_newnet` flag exists; network isolation is an nsjail feature, so outbound network from user code is unrestricted by default. Affects: T2 controls/likelihood, T5 status (unmitigated), T8.
- [Code-verified] `global_settings` is plaintext at rest under the default DB backend: `set_value_in_global_settings` stores the raw JSON value with no encryption (`windmill-common/src/global_settings.rs:259`); the encrypting secret backend (`secret_backend/database.rs:66`) only encrypts per-workspace `variable` rows with `is_secret=true`. Instance-level SMTP/OAuth/AI/object-store secrets are therefore plaintext. Affects: T6 impact/controls, T7.
- [Owner-states] Internet-facing instances are typically exposed directly with no built-in rate limiting / WAF. Affects: T16, T18 likelihood. Verify by: confirm absence of a rate-limit layer in `windmill-api/src/lib.rs` middleware stack.
- [Owner-states] Managed offering provides an optional dedicated DB/worker/namespace tier for sensitive tenants; the shared tier relies solely on the software authz boundary. Affects: T3 controls. Verify by: deployment topology (not in this repo) — out-of-tree.
- [Owner-states] Per-job rlimits/timeouts exist; instance-wide DoS by an authed tenant is risk-accepted on shared self-host. Affects: T16 status. Verify by: locate the rlimit/timeout enforcement in the worker execution path and confirm there is no global queue/concurrency cap.
- [Owner-states] Message-queue trigger authenticity is delegated to broker ACLs only. Affects: T20 status. Verify by: review `windmill-trigger-{kafka,sqs,nats,mqtt,postgres}` consume paths for any payload authentication.
## 7. Provenance
- mode: bootstrap-then-interview
- date: 2026-06-05
- target: /home/rfiszel/windmill/backend @ 819ba5e150
- inputs: git-log mined + GitHub security advisories (gh api, 73 advisories) + CHANGELOG; seed: THREAT_MODEL.md (bootstrap pass)
- owner: Ruben Fiszel (Windmill core dev)
## 8. Recommended mitigations
| mitigation | threat_ids | closes_class | effort |
|---|---|---|---|
| Centralize a single audited query-builder that forbids string-interpolated SQL; ban `format!`-built queries via lint/CI | T1 | yes | M |
| Route all outbound requests through one SSRF-guarded HTTP client (allowlist/denylist of private+metadata ranges, redirects disabled, re-validated per hop) | T2 | yes | M |
| Enforce authorization centrally in middleware (scope + RLS + folder ACL) with deny-by-default and a per-route coverage test, instead of per-handler checks | T3, T10, T14, T22 | yes | L |
| Treat all user-supplied identifiers as data: pass via argv/env/structured params, never splice into generated wrapper source; validate against strict allowlists at the boundary | T4 | yes | M |
| Make `nsjail` + network-namespace isolation default-on / fail-closed (flip `ENABLE_NSJAIL` and `clone_newnet` defaults) and remove privileged/dind defaults from shipped compose; default-deny debugger | T2, T5, T7, T8 | partial | L |
| Encrypt `global_settings` at rest under the workspace/instance key even on the default DB secret backend, so a DB read no longer yields plaintext instance-wide credentials | T6, T7 | partial | M |
| Ship hardened defaults: random per-install secrets, no default admin password, Postgres not exposed, CORS locked to configured origin, TLS-on | T7, T18, T21 | partial | M |
| Resolve secrets/resources only with the caller's identity and scope every cache entry by (caller, scope); apply uniformly to AI proxy, MCP, and exports | T6 | yes | M |
| Output-encode/sanitize all stored content at render and force `nosniff` + restrictive CSP on every user-content response | T11 | yes | M |
| Verify webhook authenticity uniformly (constant-time HMAC + timestamp/nonce anti-replay) in a shared trigger-auth helper | T12 | yes | S |
| Canonicalize + confine all file-path inputs to a base dir and never follow symlinks in log/file readers | T13 | yes | S |
| Mask secrets at the log sink and keep secrets out of worker process env (`/proc`) — pass via files/pipes scrubbed after use | T15 | partial | M |
| Add global rate limiting and per-tenant resource/queue quotas at the edge | T16, T18 | partial | M |
| Pin and integrity-verify hub scripts and CI actions; SBOM + automated base-image CVE scanning in release | T9 | partial | M |
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
da5189cf69a453de3855057f41be0d84e5910707
2c7964460327fab5e3a27c0f74b8d6f26ab7f79a
backend/oauth_connect.json
+44 -11
View File
@@ -154,28 +154,61 @@
"zoho": {
"auth_url": "https://accounts.zoho.com/oauth/v2/auth",
"token_url": "https://accounts.zoho.com/oauth/v2/token",
"scopes": [
"ZohoAssist.sessionapi.ALL"
],
"scopes": ["ZohoAssist.sessionapi.ALL"],
"extra_params": {
"access_type": "offline"
}
},
"snowflake_oauth": {},
"snowflake_oauth": {
"connect_config_template": {
"display_name": "Snowflake",
"label": "Snowflake Account Identifier",
"placeholder": "<orgname>-<account_name>",
"help_url": "https://docs.snowflake.com/en/user-guide/admin-account-identifier#using-an-account-name-as-an-identifier",
"auth_url": "https://{instance}.snowflakecomputing.com/oauth/authorize",
"token_url": "https://{instance}.snowflakecomputing.com/oauth/token-request",
"req_body_auth": false,
"extra_params_key": "account_identifier",
"resource_mapping": { "account_identifier": "{instance}" }
}
},
"apify": {
"auth_url": "https://console.apify.com/authorize/oauth",
"token_url": "https://console-backend.apify.com/oauth/apps/token",
"scopes": [
"profile",
"full_api_access"
],
"scopes": ["profile", "full_api_access"],
"extra_params": {}
},
"docusign": {
"auth_url": "https://account.docusign.com/oauth/auth",
"token_url": "https://account.docusign.com/oauth/token",
"scopes": [
"signature"
]
"scopes": ["signature"],
"sandbox": {
"auth_url": "https://account-d.docusign.com/oauth/auth",
"token_url": "https://account-d.docusign.com/oauth/token"
}
},
"salesforce": {
"auth_url": "https://login.salesforce.com/services/oauth2/authorize",
"token_url": "https://login.salesforce.com/services/oauth2/token",
"scopes": ["api", "refresh_token", "offline_access"],
"sandbox": {
"auth_url": "https://test.salesforce.com/services/oauth2/authorize",
"token_url": "https://test.salesforce.com/services/oauth2/token"
}
},
"servicenow": {
"connect_config_template": {
"display_name": "ServiceNow",
"label": "ServiceNow Instance",
"placeholder": "<instance> (e.g. dev12345)",
"help_url": "https://www.servicenow.com/docs/bundle/zurich-platform-security/page/administer/security/concept/c_OAuthApplications.html",
"auth_url": "https://{instance}.service-now.com/oauth_auth.do",
"token_url": "https://{instance}.service-now.com/oauth_token.do",
"req_body_auth": true,
"strip_suffix": ".service-now.com",
"resource_mapping": {
"instance_url": "https://{instance}.service-now.com"
}
}
}
}
backend/parsers/windmill-parser-ts/src/lib.rs
+4 -1
View File
@@ -129,7 +129,10 @@ impl Visit for ImportsFinder {
/// See also: [`parse_relative_imports`] for resolved absolute paths.
pub fn parse_expr_for_imports(code: &str, skip_type_only: bool) -> anyhow::Result<Vec<String>> {
let cm: Lrc<SourceMap> = Default::default();
let fm = cm.new_source_file(FileName::Custom("main.d.ts".into()).into(), code.to_string());
let fm = cm.new_source_file(
FileName::Custom("main.d.ts".into()).into(),
code.to_string(),
);
let mut tss = TsSyntax::default();
tss.disallow_ambiguous_jsx_like;
tss.tsx = true;
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+24 -24
View File
@@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill-common"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"aho-corasick",
"anyhow",
@@ -6263,7 +6263,7 @@ dependencies = [
[[package]]
name = "windmill-macros"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"proc-macro2",
"quote",
@@ -6275,7 +6275,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"convert_case",
"serde",
@@ -6284,7 +6284,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6296,7 +6296,7 @@ dependencies = [
[[package]]
name = "windmill-parser-csharp"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6308,7 +6308,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"gosyn",
@@ -6320,7 +6320,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6332,7 +6332,7 @@ dependencies = [
[[package]]
name = "windmill-parser-java"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6344,7 +6344,7 @@ dependencies = [
[[package]]
name = "windmill-parser-nu"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"nu-parser",
@@ -6355,7 +6355,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6366,7 +6366,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
@@ -6378,7 +6378,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-asset"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6389,7 +6389,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"async-recursion",
@@ -6411,7 +6411,7 @@ dependencies = [
[[package]]
name = "windmill-parser-r"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"serde_json",
@@ -6423,7 +6423,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ruby"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6437,7 +6437,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"convert_case",
@@ -6454,7 +6454,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6467,7 +6467,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql-asset"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"serde",
@@ -6479,7 +6479,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"lazy_static",
@@ -6497,7 +6497,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts-asset"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
@@ -6513,7 +6513,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wac"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"rustpython-ast",
@@ -6529,7 +6529,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"getrandom 0.2.17",
@@ -6561,7 +6561,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"serde",
@@ -6572,7 +6572,7 @@ dependencies = [
[[package]]
name = "windmill-types"
version = "1.708.0"
version = "1.719.0"
dependencies = [
"anyhow",
"bitflags",
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.708.0"
version = "1.719.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/src/main.rs
+42 -7
View File
@@ -57,11 +57,14 @@ use windmill_common::{
PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RESTART_COORDINATION_SETTING,
RETENTION_PERIOD_SECS_SETTING, RUBY_REPOS_SETTING, SAML_METADATA_SETTING,
SCIM_TOKEN_SETTING, SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING,
TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING,
UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING,
WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING,
WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_REGISTRIES_SETTING,
SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING,
SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING,
SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING,
STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING,
WORKSPACE_REGISTRIES_SETTING,
},
scripts::ScriptLang,
stats_oss::schedule_stats,
@@ -134,8 +137,11 @@ use crate::monitor::{
reload_job_default_timeout_setting, reload_job_isolation_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_nsjail_tmp_backing_setting,
reload_nsjail_tmpfs_size_setting, reload_otel_tracing_proxy_setting,
reload_pip_index_url_setting, reload_retention_period_setting, reload_scim_token_setting,
reload_smtp_config, reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting,
reload_pip_index_url_setting, reload_retention_period_setting,
reload_sandbox_image_cache_max_setting, reload_sandbox_image_default_registry_setting,
reload_sandbox_image_max_size_setting, reload_sandbox_image_pull_policy_setting,
reload_sandbox_registry_auth_setting, reload_scim_token_setting, reload_smtp_config,
reload_store_audit_logs_s3_setting, reload_uv_exclude_newer_setting,
reload_uv_index_strategy_setting, reload_uv_python_install_mirror_setting,
reload_worker_config, MonitorIteration,
};
@@ -258,6 +264,15 @@ pub fn main() -> anyhow::Result<()> {
}
async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
// The `cache` CLI mode never connects to the DB, so HUB_BASE_URL keeps its
// compiled default. Allow overriding it via env so the prebuild cache step can
// be pointed at a private/staging hub (e.g. a local proxy for testing).
if let Ok(hub_base_url) = std::env::var("HUB_BASE_URL") {
if !hub_base_url.is_empty() {
tracing::info!("Overriding hub base url from env: {hub_base_url}");
windmill_common::HUB_BASE_URL.store(std::sync::Arc::new(hub_base_url));
}
}
let file_path = file_path.unwrap_or("./hubPaths.json".to_string());
let mut file = File::open(&file_path)
.await
@@ -567,6 +582,7 @@ fn print_help() {
println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup");
println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool");
println!(" SYNC_CACHED_RT = false Sync cached resource types to admins workspace on server start");
println!(" HUB_BASE_URL = https://hub.windmill.dev Hub to fetch scripts from in `cache` mode (server/worker use the DB setting instead)");
println!();
println!("Notes:");
println!("- Advanced and less commonly used settings are managed via the database and are omitted here.");
@@ -1654,6 +1670,12 @@ async fn process_notify_event(
match *source_type {
"script" => {
windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key);
// Evict the relative-import latest-hash cache so a redeployed
// imported script flips the content cache to its new version
// across all replicas within a poll interval (see #6769). Keyed
// by the bare path, matching this event's payload.
windmill_api_scripts::scripts::RAW_SCRIPT_LATEST_HASH_CACHE
.remove(&format!("{workspace_id}:{path}"));
if *kind == "preprocessor" {
match sqlx::query_scalar::<_, i64>(
"SELECT fv.id
@@ -1811,6 +1833,19 @@ async fn process_notify_event(
JOB_ISOLATION_SETTING => reload_job_isolation_setting(conn).await,
NSJAIL_TMPFS_SIZE_MB_SETTING => reload_nsjail_tmpfs_size_setting(conn).await,
NSJAIL_TMP_BACKING_SETTING => reload_nsjail_tmp_backing_setting(conn).await,
SANDBOX_IMAGE_MAX_SIZE_MB_SETTING => {
reload_sandbox_image_max_size_setting(conn).await
}
SANDBOX_IMAGE_CACHE_MAX_MB_SETTING => {
reload_sandbox_image_cache_max_setting(conn).await
}
SANDBOX_IMAGE_PULL_POLICY_SETTING => {
reload_sandbox_image_pull_policy_setting(conn).await
}
SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING => {
reload_sandbox_image_default_registry_setting(conn).await
}
SANDBOX_REGISTRY_AUTH_SETTING => reload_sandbox_registry_auth_setting(conn).await,
#[cfg(feature = "parquet")]
OBJECT_STORE_CONFIG_SETTING => {
if !disable_s3_store {
backend/src/monitor.rs
+93 -3
View File
@@ -66,7 +66,9 @@ use windmill_common::{
OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING,
POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING,
REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING,
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SANDBOX_IMAGE_CACHE_MAX_MB_SETTING,
SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING,
SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING,
STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
@@ -112,8 +114,10 @@ use windmill_worker::{
JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML,
NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB,
NSJAIL_TMP_BACKING, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER,
UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES,
PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, SANDBOX_IMAGE_CACHE_MAX_MB,
SANDBOX_IMAGE_DEFAULT_REGISTRY, SANDBOX_IMAGE_MAX_SIZE_MB, SANDBOX_IMAGE_PULL_POLICY,
SANDBOX_REGISTRY_AUTH, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY,
UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES,
};
#[cfg(feature = "parquet")]
@@ -407,6 +411,11 @@ pub async fn initial_load(
reload_job_isolation_setting(&conn).await;
reload_nsjail_tmpfs_size_setting(&conn).await;
reload_nsjail_tmp_backing_setting(&conn).await;
reload_sandbox_image_max_size_setting(&conn).await;
reload_sandbox_image_cache_max_setting(&conn).await;
reload_sandbox_image_pull_policy_setting(&conn).await;
reload_sandbox_image_default_registry_setting(&conn).await;
reload_sandbox_registry_auth_setting(&conn).await;
reload_extra_pip_index_url_setting(&conn).await;
reload_pip_index_url_setting(&conn).await;
reload_uv_index_strategy_setting(&conn).await;
@@ -2045,6 +2054,66 @@ pub async fn reload_nsjail_tmp_backing_setting(conn: &Connection) {
.await;
}
pub async fn reload_sandbox_image_max_size_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
SANDBOX_IMAGE_MAX_SIZE_MB_SETTING,
"SANDBOX_IMAGE_MAX_SIZE_MB",
SANDBOX_IMAGE_MAX_SIZE_MB.clone(),
)
.await;
}
pub async fn reload_sandbox_image_cache_max_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
SANDBOX_IMAGE_CACHE_MAX_MB_SETTING,
"SANDBOX_IMAGE_CACHE_MAX_MB",
SANDBOX_IMAGE_CACHE_MAX_MB.clone(),
)
.await;
}
pub async fn reload_sandbox_image_pull_policy_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
SANDBOX_IMAGE_PULL_POLICY_SETTING,
"SANDBOX_IMAGE_PULL_POLICY",
SANDBOX_IMAGE_PULL_POLICY.clone(),
)
.await;
}
pub async fn reload_sandbox_image_default_registry_setting(conn: &Connection) {
reload_option_setting_with_tracing(
conn,
SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING,
"SANDBOX_IMAGE_DEFAULT_REGISTRY",
SANDBOX_IMAGE_DEFAULT_REGISTRY.clone(),
)
.await;
}
pub async fn reload_sandbox_registry_auth_setting(conn: &Connection) {
// Secret-aware: the value is a raw docker/podman auth.json with credentials, so
// it must never be logged. Load directly (the generic reload_option_setting path
// logs the value via load_option_setting_value) and only log a redacted message.
let q =
match load_value_from_global_settings_with_conn(conn, SANDBOX_REGISTRY_AUTH_SETTING, true)
.await
{
Ok(q) => q,
Err(e) => {
tracing::error!("Error reloading setting SANDBOX_REGISTRY_AUTH: {e:?}");
return;
}
};
let value = q.and_then(|q| serde_json::from_value::<String>(q).ok());
let configured = value.as_ref().is_some_and(|v| !v.trim().is_empty());
*SANDBOX_REGISTRY_AUTH.write().await = value;
tracing::info!("Loaded setting SANDBOX_REGISTRY_AUTH (redacted), configured={configured}");
}
pub async fn reload_job_isolation_setting(conn: &Connection) {
let value =
match load_value_from_global_settings_with_conn(conn, JOB_ISOLATION_SETTING, true).await {
@@ -2705,6 +2774,26 @@ pub async fn monitor_db(
}
};
// run every hour (120 iterations * 30s = 3600s)
let cleanup_stale_server_heartbeats_f = async {
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) {
if let Some(db) = conn.as_sql() {
match windmill_api::cleanup_stale_server_heartbeats(db).await {
Ok(count) if count > 0 => {
tracing::info!(
"Deleted {} stale server_heartbeat background_task_state rows",
count
);
}
Err(e) => {
tracing::error!("Error cleaning up stale server_heartbeat rows: {:?}", e);
}
_ => {}
}
}
}
};
// run every hour (120 iterations * 30s = 3600s)
let manage_audit_partitions_f = async {
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(120) {
@@ -2763,6 +2852,7 @@ pub async fn monitor_db(
native_triggers_sync_f,
cleanup_notify_events_f,
check_expiring_tokens_f,
cleanup_stale_server_heartbeats_f,
manage_audit_partitions_f,
export_audit_logs_to_object_store_f,
cleanup_scheduled_job_deletions_f,
backend/tests/dependency_map.rs
+2
View File
@@ -451,6 +451,7 @@ def main():
preserve_on_behalf_of: None,
ws_error_handler_muted: None,
labels: None,
skip_draft_deletion: None,
})
.send()
.await
@@ -513,6 +514,7 @@ def main():
custom_path: None,
preserve_on_behalf_of: None,
labels: None,
skip_draft_deletion: None,
})
.send()
.await
backend/tests/fixtures/jobs_read_auth.sql
Vendored
+192
View File
@@ -0,0 +1,192 @@
-- Fixture for the single-job read authorization regression test
-- (see tests/jobs_read_auth.rs).
--
-- Users available from `base`:
-- test-user (admin, token SECRET_TOKEN)
-- test-user-2 (User, token SECRET_TOKEN_2) -- owner of the secret script
-- test-user-3 (User, token SECRET_TOKEN_3) -- the unprivileged "viewer"
--
-- test-user-3 is NOT a member of any folder/group granting access to
-- `u/test-user-2/...`, so under the same RLS as `jobs/list` they cannot see any
-- of these jobs unless they created them.
-- A tag-scoped token for test-user-2 (who can read both VICTIM (tag 'deno') and
-- the flow (tag 'flow')). The `if_jobs:filter_tags:deno` modifier restricts it to
-- the 'deno' tag, so it must NOT be able to mint a share token for the 'flow' job.
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES (
encode(sha256('SCOPED_DENO_TOKEN'::bytea), 'hex'), 'SCOPED_DEN', 'SCOPED_DENO_TOKEN',
'test2@windmill.dev', 'scoped deno token', false,
ARRAY['jobs:read', 'if_jobs:filter_tags:deno']
);
-- RUNNING job: queued (no completed row) and owned by test-user-2. Used to check
-- that `completed/get_result_maybe?get_started=true` authorizes before disclosing
-- running-state to a non-reader.
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner
) VALUES (
'77777777-7777-7777-7777-777777777777', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/running_secret', 'deno', true
);
INSERT INTO public.v2_job_queue (id, workspace_id, scheduled_for, running, tag) VALUES
('77777777-7777-7777-7777-777777777777', 'test-workspace', '2023-01-01 00:00:00', true, 'deno');
-- 1. VICTIM job: a completed run of test-user-2's private script, e.g. produced
-- by a public HTTP trigger. `created_by` is the route identity (test-user-2),
-- NOT the viewer; `permissioned_as`/`runnable_path` sit in test-user-2's
-- namespace; `visible_to_owner` is true. Its args + result carry secrets.
-- Pre-fix, test-user-3 could read all of these by UUID.
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner, args
) VALUES (
'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/secret_script', 'deno', true,
'{"secret": "LEAK_TEST_ARGS"}'
);
INSERT INTO public.v2_job_completed (
id, workspace_id, duration_ms, status, result
) VALUES (
'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'test-workspace', 1000,
'success'::job_status, '{"secret": "RESULT_SECRET"}'
);
INSERT INTO public.job_logs (job_id, workspace_id, logs) VALUES
('aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', 'test-workspace', 'secret logs LEAK_TEST_LOGS');
-- 2. APP-style job: run by the viewer (test-user-3) on behalf of an app whose
-- policy executes as test-user-2. `created_by` is the launching viewer, but
-- `permissioned_as`/`runnable_path` are the app owner's and
-- `visible_to_owner` is false (apps hide their component runs from the runs
-- list). This is the case that must KEEP working after the fix: the viewer
-- polls their own component result by UUID.
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner, args
) VALUES (
'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'test-workspace', 'test-user-3',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/app_component', 'deno', false,
'{"app_arg": "ok"}'
);
INSERT INTO public.v2_job_completed (
id, workspace_id, duration_ms, status, result
) VALUES (
'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb', 'test-workspace', 1000,
'success'::job_status, '{"app_result": "visible_to_launcher"}'
);
-- 3. ANONYMOUS job: a public-trigger run whose creator is `anonymous`. Reading
-- it without authentication must keep working (unchanged behavior).
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner, args
) VALUES (
'cccccccc-cccc-cccc-cccc-cccccccccccc', 'test-workspace', 'anonymous',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/public_trigger', 'deno', true,
'{"public": "arg"}'
);
INSERT INTO public.v2_job_completed (
id, workspace_id, duration_ms, status, result
) VALUES (
'cccccccc-cccc-cccc-cccc-cccccccccccc', 'test-workspace', 1000,
'success'::job_status, '{"public": "result"}'
);
-- 4. FLOW + STEP: test-user-3 has *read* access to folder `shared` (extra_perms),
-- so they can see flow `f/shared/flow1` (run by test-user-2) even though they
-- did not launch it. The flow's STEP job runs the inner script
-- `u/test-user-2/inner_secret` (test-user-3 has NO direct ACL on it) and is
-- not in their list. Visibility must be INHERITED from the flow root: being
-- able to see the flow means being able to inspect its steps (the flow-run UI
-- fetches each step by id). This guards against the fix over-blocking.
INSERT INTO public.folder (workspace_id, name, display_name, owners, extra_perms, created_by)
VALUES ('test-workspace', 'shared', 'Shared Folder', '{"u/test-user-2"}',
'{"u/test-user-3": false}', 'test-user-2');
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner
) VALUES (
'dddddddd-dddd-dddd-dddd-dddddddddddd', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'flow', 'deno', 'f/shared/flow1', 'flow', true
);
INSERT INTO public.v2_job_completed (
id, workspace_id, duration_ms, status, result
) VALUES (
'dddddddd-dddd-dddd-dddd-dddddddddddd', 'test-workspace', 1000,
'success'::job_status, '{"flow": "done"}'
);
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner,
parent_job, root_job, flow_innermost_root_job, args
) VALUES (
'eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/inner_secret', 'deno', true,
'dddddddd-dddd-dddd-dddd-dddddddddddd', 'dddddddd-dddd-dddd-dddd-dddddddddddd',
'dddddddd-dddd-dddd-dddd-dddddddddddd', '{"step_arg": "x"}'
);
INSERT INTO public.v2_job_completed (
id, workspace_id, duration_ms, status, result
) VALUES (
'eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee', 'test-workspace', 1000,
'success'::job_status, '{"step": "STEP_RESULT_INHERITED"}'
);
-- 5. DEEP NESTING / MIDDLE-LAYER VISIBILITY: top flow `f/secret/top` is NOT
-- visible to test-user-3; it has a sub-flow step `f/shared/mid` that IS visible
-- (folder `shared`); and that sub-flow has its own leaf step running
-- `u/test-user-2/deep_secret` (not visible). The leaf's `root_job` points at the
-- *outermost* top (not visible), so visibility must come from the *intermediate*
-- sub-flow the user can see — which requires walking the full parent chain, not
-- just [self, root].
INSERT INTO public.folder (workspace_id, name, display_name, owners, extra_perms, created_by)
VALUES ('test-workspace', 'secret', 'Secret Folder', '{"u/test-user-2"}', '{}', 'test-user-2');
-- top flow (not visible to test-user-3)
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner
) VALUES (
'ffffffff-ffff-ffff-ffff-ffffffffffff', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'flow', 'deno', 'f/secret/top', 'flow', true
);
-- intermediate sub-flow (visible via folder `shared`), child of top
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner,
parent_job, root_job, flow_innermost_root_job
) VALUES (
'99999999-9999-9999-9999-999999999999', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'flow', 'deno', 'f/shared/mid', 'flow', true,
'ffffffff-ffff-ffff-ffff-ffffffffffff', 'ffffffff-ffff-ffff-ffff-ffffffffffff',
'ffffffff-ffff-ffff-ffff-ffffffffffff'
);
-- leaf step of the sub-flow; runnable not visible, root_job = outermost top (not visible)
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, script_lang, runnable_path, tag, visible_to_owner,
parent_job, root_job, flow_innermost_root_job
) VALUES (
'88888888-8888-8888-8888-888888888888', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'script', 'deno', 'u/test-user-2/deep_secret', 'deno', true,
'99999999-9999-9999-9999-999999999999', 'ffffffff-ffff-ffff-ffff-ffffffffffff',
'99999999-9999-9999-9999-999999999999'
);
INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES
('ffffffff-ffff-ffff-ffff-ffffffffffff', 'test-workspace', 1000, 'success'::job_status,
'{"top": "TOP_SECRET_RESULT"}'),
('99999999-9999-9999-9999-999999999999', 'test-workspace', 1000, 'success'::job_status,
'{"mid": "MID_RESULT"}'),
('88888888-8888-8888-8888-888888888888', 'test-workspace', 1000, 'success'::job_status,
'{"deep": "DEEP_STEP_INHERITED"}');
backend/tests/fixtures/mcp_token_exfil.sql
Vendored
+29
View File
@@ -0,0 +1,29 @@
-- Fixture for the MCP token-exfiltration regression test.
--
-- Models a malicious developer (test-user-3, a plain workspace member) who:
-- - owns an MCP resource they are allowed to read, and
-- - points that resource's `token` field at a secret variable living in a
-- folder they have NO access to (`f/locked`, only test-user/admin owns it).
--
-- The secret variable `f/locked/secret_token` itself is inserted by the test in
-- Rust (so it is encrypted with the real workspace key); this fixture only sets
-- up the locked folder, the resource, and their permissions.
-- Folder the developer cannot read (empty extra_perms, owned by admin only).
INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by)
VALUES ('test-workspace', 'locked', 'Locked Folder', '{"u/test-user"}', '{}', 'test-user');
-- MCP resource owned by the developer (so RLS lets them read the resource),
-- whose token references the locked secret. The URL is a non-resolvable public
-- host so that, for an authorized caller, resolution succeeds but the later
-- connection/SSRF step fails deterministically without network access.
INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by)
VALUES (
'test-workspace',
'u/test-user-3/evil_mcp',
'{"name": "evil", "url": "https://mcp.invalid.windmill.test", "token": "$var:f/locked/secret_token"}',
'MCP resource whose token points at a locked secret',
'mcp',
'{}',
'test-user-3'
);
backend/tests/flow_engine_parity.rs
+169
View File
@@ -2916,6 +2916,7 @@ export function main() {
expr: "flow_env.STOP === true".to_string(),
skip_if_stopped: true,
error_message: None,
error_include_result: false,
});
m
};
@@ -2966,6 +2967,92 @@ export function main() {
Ok(())
}
// stop_after_if with `error_message` + `error_include_result` should fail the
// flow but preserve the stopping step's own result inside the raised error
// object, i.e. `{ "error": { .., "result": <step result> } }`. With the flag off
// (the default) the error object carries no `result`. Regression for the
// early-stop branch in `update_flow_status_after_job_completion_internal`.
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_stop_after_if_error_include_result(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let make_flow = |include_result: bool| {
let mut m = flow_module(
"step",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
content: r#"
export function main() {
return { userErrors: ["email taken"], ok: false };
}
"#
.to_string(),
path: None,
lock: None,
tag: None,
concurrency_settings: Default::default(),
is_trigger: None,
assets: None,
},
);
m.stop_after_if = Some(windmill_common::flows::StopAfterIf {
expr: "true".to_string(),
skip_if_stopped: false,
error_message: Some("API returned userErrors".to_string()),
error_include_result: include_result,
});
FlowValue { modules: vec![m], same_worker: false, ..Default::default() }
};
// include_result = true: result preserves both the error and the step output
let job = RunJob::from(JobPayload::RawFlow {
value: make_flow(true),
path: None,
restarted_from: None,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(
!job.success,
"flow with raised early-stop error should fail"
);
let result = job.json_result().unwrap();
assert_eq!(
result["error"]["name"], "EarlyStopError",
"expected EarlyStopError; got {result:?}"
);
assert_eq!(result["error"]["message"], "API returned userErrors");
assert_eq!(
result["error"]["result"],
json!({ "userErrors": ["email taken"], "ok": false }),
"step result should be preserved under `error.result`; got {result:?}"
);
// include_result = false (default behavior): result is the bare error object
let job = RunJob::from(JobPayload::RawFlow {
value: make_flow(false),
path: None,
restarted_from: None,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(
!job.success,
"flow with raised early-stop error should fail"
);
let result = job.json_result().unwrap();
assert_eq!(result["error"]["name"], "EarlyStopError");
assert!(
result["error"].get("result").is_none(),
"without the flag the error must not embed the step result; got {result:?}"
);
Ok(())
}
// retry_if predicate sees flow_env. Regression for the two evaluate_retry
// call sites in `update_flow_status_after_job_completion_internal` (lines
// 1194 and 1576) which used to pass `None` for flow_env.
@@ -3093,6 +3180,7 @@ export function main(i: number) {
expr: "flow_env.STOP === true".to_string(),
skip_if_stopped: true,
error_message: None,
error_include_result: false,
});
m
};
@@ -3143,3 +3231,84 @@ export function main() {
Ok(())
}
// stop_after_all_iters_if with `error_message` + `error_include_result` fails the
// flow and embeds the loop's aggregated iteration results under `error.result`.
// Covers the loop/branch-all path where `nresult` is already populated with the
// aggregated results (distinct from the per-step fallback to `result`).
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_stop_after_all_iters_if_error_includes_result(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let inner = flow_module(
"iter_step",
FlowModuleValue::RawScript {
input_transforms: [js_input("i", "flow_input.iter.value")].into(),
language: ScriptLang::Deno,
content: r#"
export function main(i: number) {
return { iter: i };
}
"#
.to_string(),
path: None,
lock: None,
tag: None,
concurrency_settings: Default::default(),
is_trigger: None,
assets: None,
},
);
let loop_module = {
let mut m = flow_module(
"loop",
FlowModuleValue::ForloopFlow {
iterator: InputTransform::Javascript { expr: "[1, 2, 3]".to_string() },
modules: vec![inner],
modules_node: None,
skip_failures: false,
parallel: false,
parallelism: None,
squash: None,
},
);
m.stop_after_all_iters_if = Some(windmill_common::flows::StopAfterIf {
expr: "true".to_string(),
skip_if_stopped: false,
error_message: Some("loop failed".to_string()),
error_include_result: true,
});
m
};
let flow = FlowValue { modules: vec![loop_module], same_worker: false, ..Default::default() };
let job = RunJob::from(JobPayload::RawFlow { value: flow, path: None, restarted_from: None })
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(
!job.success,
"loop with a raised early-stop error should fail"
);
let result = job.json_result().unwrap();
assert_eq!(result["error"]["name"], "EarlyStopError", "got {result:?}");
assert_eq!(result["error"]["message"], "loop failed");
// error.result holds the aggregated iteration results (one per iteration)
let iters = result["error"]["result"].as_array().unwrap_or_else(|| {
panic!("error.result should be an array of iteration results; got {result:?}")
});
let iter_values: Vec<_> = iters.iter().map(|r| r["iter"].clone()).collect();
assert_eq!(
iter_values,
vec![json!(1), json!(2), json!(3)],
"error.result should contain each iteration's output; got {result:?}"
);
Ok(())
}
backend/tests/jobs_read_auth.rs
+512
View File
@@ -0,0 +1,512 @@
//! Regression test for the single-job read authorization bypass.
//!
//! The single-job read endpoints (`/jobs_u/get`, `/completed/get`,
//! `/completed/get_result`, `/get_args`, `/get_logs`, `/getupdate`, ...) fetch a
//! job through the root DB handle, filtered only by job id + workspace. That is
//! required for the unauthenticated approval / public-trigger / anonymous-job
//! flows, but for a *logged-in* user it meant any workspace member — including a
//! plain viewer with no ACL on the runnable — could read another user's job
//! args/result/logs simply by obtaining the job UUID, even though the same job is
//! hidden from them in `jobs/list` (RLS-filtered) and the underlying script
//! returns 404.
//!
//! The fix (`require_job_read_access`) gates the authenticated case: a caller may
//! read a job they created (covers app components / webhooks / their own runs)
//! or one visible to them under the same RLS as `jobs/list` (admins bypass);
//! otherwise 404. Unauthenticated access is unchanged (anonymous jobs only).
//!
//! This test pins down, against the `jobs_read_auth` fixture:
//! - a viewer is denied the victim job's full record / result / result_maybe /
//! args / logs / live update by UUID, and the secret never appears in the
//! body (the core fix; pre-fix these returned 200 with the secret),
//! - the job's owner and an admin can still read it (no over-blocking),
//! - the "app component" affordance survives: a viewer who *launched* a job
//! (created_by) running as someone else's identity can still read its result,
//! - unauthenticated behavior is unchanged: anonymous jobs readable, the
//! non-anonymous victim job rejected.
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
const VICTIM: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const APP_JOB: &str = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
const ANON_JOB: &str = "cccccccc-cccc-cccc-cccc-cccccccccccc";
const FLOW_JOB: &str = "dddddddd-dddd-dddd-dddd-dddddddddddd";
const STEP_JOB: &str = "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee";
// Deep nesting: top (not visible) -> mid (visible via folder) -> deep leaf.
const TOP_SECRET_FLOW: &str = "ffffffff-ffff-ffff-ffff-ffffffffffff";
const DEEP_LEAF_JOB: &str = "88888888-8888-8888-8888-888888888888";
// A queued/running job (no completed row) owned by test-user-2.
const RUNNING_JOB: &str = "77777777-7777-7777-7777-777777777777";
// Secrets that must never leak to an unauthorized viewer.
const RESULT_SECRET: &str = "RESULT_SECRET";
const ARGS_SECRET: &str = "LEAK_TEST_ARGS";
const LOGS_SECRET: &str = "LEAK_TEST_LOGS";
fn client() -> reqwest::Client {
reqwest::Client::new()
}
async fn get(base: &str, path: &str, token: Option<&str>) -> (reqwest::StatusCode, String) {
let mut req = client().get(format!("{base}/{path}"));
if let Some(token) = token {
req = req.header("Authorization", format!("Bearer {token}"));
}
let resp = req.send().await.expect("request");
let status = resp.status();
let body = resp.text().await.expect("body");
(status, body)
}
#[sqlx::test(fixtures("base", "jobs_read_auth"))]
async fn test_single_job_read_authorization(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/jobs_u");
// result_by_id / get_otel_traces live on the authed `/jobs` service, not `/jobs_u`.
let authed_base = format!("http://localhost:{port}/api/w/test-workspace/jobs");
// The endpoints that return the victim job's sensitive data by UUID.
let endpoints = [
("get", format!("get/{VICTIM}")),
("completed/get", format!("completed/get/{VICTIM}")),
(
"completed/get_result",
format!("completed/get_result/{VICTIM}"),
),
(
"completed/get_result_maybe",
format!("completed/get_result_maybe/{VICTIM}"),
),
("get_args", format!("get_args/{VICTIM}")),
("get_logs", format!("get_logs/{VICTIM}")),
(
"get_completed_logs_tail",
format!("get_completed_logs_tail/{VICTIM}"),
),
("get_flow_all_logs", format!("get_flow_all_logs/{VICTIM}")),
(
"completed/get_timing",
format!("completed/get_timing/{VICTIM}"),
),
("getupdate", format!("getupdate/{VICTIM}?only_result=true")),
];
// ---- CORE REGRESSION: the viewer (test-user-3) is denied on every endpoint
// and no secret ever appears in the body. Pre-fix these returned 200
// and leaked the secret.
for (name, path) in &endpoints {
let (status, body) = get(&base, path, Some("SECRET_TOKEN_3")).await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"viewer must get 403 on {name} (got {status}): {body}"
);
for secret in [RESULT_SECRET, ARGS_SECRET, LOGS_SECRET] {
assert!(
!body.contains(secret),
"viewer response for {name} leaked `{secret}`: {body}"
);
}
}
// The 403 for an existing-but-forbidden job carries actionable guidance
// (request a share link), distinguishing it from a plain not-found.
let (status, body) = get(
&base,
&format!("completed/get_result/{VICTIM}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(status, reqwest::StatusCode::FORBIDDEN);
assert!(
body.to_lowercase().contains("share"),
"403 body should guide the user to request a share link: {body}"
);
// A genuinely non-existent job is a 404, not a 403 — existence is only disclosed
// for jobs that actually exist in the workspace.
let missing = "00000000-0000-4000-8000-000000000000";
let (status, _) = get(
&base,
&format!("completed/get_result/{missing}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"a non-existent job must be 404, not 403 (got {status})"
);
// ---- NO OVER-BLOCKING: the job's owner (test-user-2) can read its result.
let (status, body) = get(
&base,
&format!("completed/get_result/{VICTIM}"),
Some("SECRET_TOKEN_2"),
)
.await;
assert!(
status.is_success(),
"owner must still read their own job result (got {status}): {body}"
);
assert!(
body.contains(RESULT_SECRET),
"owner result must contain the value: {body}"
);
// ---- ADMIN BYPASS: an admin (test-user) can read any job in the workspace.
let (status, body) = get(
&base,
&format!("completed/get_result/{VICTIM}"),
Some("SECRET_TOKEN"),
)
.await;
assert!(
status.is_success(),
"admin must read any job (got {status}): {body}"
);
assert!(body.contains(RESULT_SECRET), "admin result body: {body}");
// ---- APP AFFORDANCE: a viewer who LAUNCHED a job (created_by = viewer) that
// runs as another identity (permissioned_as = test-user-2,
// visible_to_owner = false) can still read its result. This is the app
// component-polling path; the fix must not break it.
let (status, body) = get(
&base,
&format!("completed/get_result/{APP_JOB}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert!(
status.is_success(),
"launcher must read a job they created even without ACL on the runnable (got {status}): {body}"
);
assert!(
body.contains("visible_to_launcher"),
"launcher should get the result they polled: {body}"
);
// ---- AUTHED `/jobs` endpoints in the same class: result_by_id (flow node
// result) and get_otel_traces (job telemetry). The viewer must be denied
// the victim by UUID. The auth gate runs before result/trace resolution,
// so 404 here is the gate, not incidental resolution failure.
let (status, body) = get(
&authed_base,
&format!("result_by_id/{VICTIM}/somenode"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"viewer must get 403 on result_by_id (got {status}): {body}"
);
assert!(!body.contains(RESULT_SECRET), "result_by_id leaked: {body}");
let (status, body) = get(
&authed_base,
&format!("get_otel_traces/{VICTIM}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"viewer must get 403 on get_otel_traces (got {status}): {body}"
);
// ---- FLOW VISIBILITY INHERITANCE: test-user-3 has folder ACL on the flow
// `f/shared/flow1` (run by test-user-2) but did NOT launch it, and has no
// ACL on the step's inner runnable `u/test-user-2/inner_secret`. They must
// still be able to (a) read the flow they can see, and (b) inspect its
// step result — visibility is inherited from the flow root. A naive
// "same as list" gate would 404 the step and break the flow-run UI.
let (status, body) = get(
&base,
&format!("completed/get_result/{FLOW_JOB}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert!(
status.is_success(),
"viewer with folder ACL must read the flow they can see (got {status}): {body}"
);
let (status, body) = get(
&base,
&format!("completed/get_result/{STEP_JOB}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert!(
status.is_success(),
"viewer must inspect a step of a flow they can see, even without ACL on the step's runnable (got {status}): {body}"
);
assert!(
body.contains("STEP_RESULT_INHERITED"),
"step result should be returned via flow-root inheritance: {body}"
);
// ---- DEEP NESTING / MIDDLE-LAYER VISIBILITY: the deep leaf's root_job is the
// top flow (NOT visible to test-user-3), but an intermediate sub-flow
// (f/shared/mid) IS visible. Reading the leaf must succeed via that middle
// ancestor — i.e. the full parent chain is walked, not just [self, root].
let (status, body) = get(
&base,
&format!("completed/get_result/{DEEP_LEAF_JOB}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert!(
status.is_success(),
"deep leaf must be readable via a visible intermediate sub-flow (got {status}): {body}"
);
assert!(
body.contains("DEEP_STEP_INHERITED"),
"deep leaf result should be returned via mid-ancestor visibility: {body}"
);
// ...but the top flow itself, in a folder the viewer cannot read, stays denied.
let (status, body) = get(
&base,
&format!("completed/get_result/{TOP_SECRET_FLOW}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"top flow in an unreadable folder must stay denied (got {status}): {body}"
);
// ---- UNAUTHENTICATED, unchanged: an anonymous-created job is readable
// without a token (public trigger / public app result polling).
let (status, body) = get(&base, &format!("completed/get_result/{ANON_JOB}"), None).await;
assert!(
status.is_success(),
"anonymous job must remain readable unauthenticated (got {status}): {body}"
);
// ---- UNAUTHENTICATED, unchanged: the non-anonymous victim job is rejected
// for an unauthenticated caller (400, the pre-existing guard).
let (status, body) = get(&base, &format!("completed/get_result/{VICTIM}"), None).await;
assert_eq!(
status,
reqwest::StatusCode::BAD_REQUEST,
"unauthenticated access to a non-anonymous job must stay rejected (got {status}): {body}"
);
assert!(
!body.contains(RESULT_SECRET),
"unauth body must not leak: {body}"
);
// ---- SHARE READ LINK (view_token) ----
// The owner (test-user-2) mints a share token for the victim job.
let (status, mint_body) = get(
&authed_base,
&format!("job_view_token/{VICTIM}"),
Some("SECRET_TOKEN_2"),
)
.await;
assert!(
status.is_success(),
"owner must be able to mint a share token (got {status}): {mint_body}"
);
let token = mint_body.trim().trim_matches('"').to_string();
assert!(
token.starts_with(VICTIM),
"token must encode the job id: {token}"
);
// The viewer (no ACL) can now read the victim job via the share link.
let (status, body) = get(
&base,
&format!("completed/get_result/{VICTIM}?view_token={token}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert!(
status.is_success(),
"view_token must grant the viewer read of the shared job (got {status}): {body}"
);
assert!(
body.contains(RESULT_SECRET),
"shared job result must be returned with a valid view_token: {body}"
);
// ...and its args/logs too (whole detail page).
let (status, _) = get(
&base,
&format!("get_args/{VICTIM}?view_token={token}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert!(
status.is_success(),
"view_token must also grant args (got {status})"
);
// The token is scoped: it does NOT authorize an unrelated job.
let (status, _) = get(
&base,
&format!("completed/get_result/{ANON_JOB}?view_token={token}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"a victim-scoped token must not authorize a different job (got {status})"
);
// A garbage token is rejected (falls through to the normal 404).
let (status, _) = get(
&base,
&format!("completed/get_result/{VICTIM}?view_token={VICTIM}.deadbeef"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"an invalid view_token must not grant access (got {status})"
);
// A share token authorizes the shared job's whole flow subtree: the owner mints
// for the top secret flow, and the viewer can then read its deep leaf.
let (status, mint_body) = get(
&authed_base,
&format!("job_view_token/{TOP_SECRET_FLOW}"),
Some("SECRET_TOKEN_2"),
)
.await;
assert!(
status.is_success(),
"owner mints token for top flow (got {status}): {mint_body}"
);
let top_token = mint_body.trim().trim_matches('"').to_string();
let (status, body) = get(
&base,
&format!("completed/get_result/{DEEP_LEAF_JOB}?view_token={top_token}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert!(
status.is_success(),
"a flow's share token must authorize its deep descendants (got {status}): {body}"
);
// A viewer who cannot read a job cannot mint a share token for it.
let (status, _) = get(
&authed_base,
&format!("job_view_token/{TOP_SECRET_FLOW}"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"a non-reader must not be able to mint a share token (got {status})"
);
// ---- TAG-SCOPED token must not mint a token outside its allowed tags ----
// SCOPED_DENO_TOKEN (test-user-2, scope `if_jobs:filter_tags:deno`) can read both
// VICTIM (tag deno) and FLOW_JOB (tag flow) by RLS, but minting must honor the
// tag scope: allowed for the deno job, denied for the flow job.
let (status, body) = get(
&authed_base,
&format!("job_view_token/{VICTIM}"),
Some("SCOPED_DENO_TOKEN"),
)
.await;
assert!(
status.is_success(),
"tag-scoped token may mint for an in-scope (deno) job (got {status}): {body}"
);
let (status, _) = get(
&authed_base,
&format!("job_view_token/{FLOW_JOB}"),
Some("SCOPED_DENO_TOKEN"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"tag-scoped token must NOT mint for an out-of-scope (flow) job (got {status})"
);
// ---- USE side: a tag-scoped token must not use someone else's valid view_token
// to read an out-of-scope job, even via handlers that don't tag-filter their
// data query (result_by_id, get_otel_traces, get_flow_debug_info). ----
// An unscoped owner mints a valid token for the flow (tag 'flow').
let (status, mint_body) = get(
&authed_base,
&format!("job_view_token/{FLOW_JOB}"),
Some("SECRET_TOKEN_2"),
)
.await;
assert!(
status.is_success(),
"owner mints flow token (got {status}): {mint_body}"
);
let flow_token = mint_body.trim().trim_matches('"').to_string();
// The deno-scoped token presents that valid flow token to the non-tag-filtered
// endpoints — must still be denied (flow tag is out of its scope).
for path in [
format!("get_otel_traces/{FLOW_JOB}?view_token={flow_token}"),
format!("result_by_id/{FLOW_JOB}/somenode?view_token={flow_token}"),
] {
let (status, _) = get(&authed_base, &path, Some("SCOPED_DENO_TOKEN")).await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"tag-scoped token must not use a view_token to read an out-of-scope job ({path}, got {status})"
);
}
// ...but the deno-scoped token CAN use an in-scope (deno) view_token.
let (status, body) = get(
&base,
&format!("completed/get_result/{VICTIM}?view_token={token}"),
Some("SCOPED_DENO_TOKEN"),
)
.await;
assert!(
status.is_success(),
"tag-scoped token may use a view_token for an in-scope (deno) job (got {status}): {body}"
);
// ---- get_result_maybe?get_started=true must authorize before disclosing the
// running-state of a queued (not-yet-completed) private job. ----
// Viewer (no ACL) must be denied rather than told the job is started.
let (status, body) = get(
&base,
&format!("completed/get_result_maybe/{RUNNING_JOB}?get_started=true"),
Some("SECRET_TOKEN_3"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::FORBIDDEN,
"viewer must be denied the running-state of a private queued job (got {status}): {body}"
);
assert!(
!body.contains("\"started\""),
"denied response must not disclose started-state: {body}"
);
// The owner still gets the in-progress response.
let (status, body) = get(
&base,
&format!("completed/get_result_maybe/{RUNNING_JOB}?get_started=true"),
Some("SECRET_TOKEN_2"),
)
.await;
assert!(
status.is_success() && body.contains("\"started\":true"),
"owner must see the running job as started (got {status}): {body}"
);
Ok(())
}
backend/tests/mcp_token_exfil.rs
+111
View File
@@ -0,0 +1,111 @@
//! Regression test for the MCP token-exfiltration vulnerability.
//!
//! `GET /api/w/{w}/resources/mcp_tools/{path}` builds an MCP client from a
//! resource whose `token` field is a `$var:` reference. Before the fix the token
//! was resolved with `get_secret_value_as_admin` on the bare DB pool — no RLS,
//! no audit — so any workspace member who could read an MCP *resource* could
//! point its token at *any* secret variable in the workspace (e.g. one in an
//! admin-only folder) and have it decrypted and shipped as a bearer token.
//!
//! The fix resolves the token through the caller's permissioned path
//! (`get_value_internal` over the authed `user_db`), so the variable RLS — the
//! same gate as `variables/get_value` — applies and the secret read is audited.
//!
//! This test pins, against the `mcp_token_exfil` fixture:
//! - a plain developer (test-user-3) who can read the MCP resource but has no
//! access to the locked secret is DENIED (401) at token resolution, before
//! any connection is attempted, and the secret never leaks;
//! - an admin (test-user) clears the variable-RLS gate, the token resolves,
//! and the request only fails later at the connect/SSRF step — proving the
//! legitimate path still resolves the token (no over-blocking).
//!
//! SSRF rejection of an author-controlled URL is covered by the unit test in
//! `windmill-mcp` (`from_resource_rejects_ssrf_url`).
#![cfg(feature = "mcp")]
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
const SECRET_VALUE: &str = "S3CRET-MCP-TOKEN-VALUE";
fn client() -> reqwest::Client {
reqwest::Client::new()
}
async fn get(base: &str, path: &str, token: &str) -> (reqwest::StatusCode, String) {
let resp = client()
.get(format!("{base}/{path}"))
.header("Authorization", format!("Bearer {token}"))
.send()
.await
.expect("request");
let status = resp.status();
let body = resp.text().await.expect("body");
(status, body)
}
#[sqlx::test(fixtures("base", "mcp_token_exfil"))]
async fn test_mcp_token_not_exfiltrated(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
// Insert the locked secret variable with a real, workspace-key-encrypted
// value so an authorized read genuinely decrypts it.
let mc = windmill_common::variables::build_crypt(&db, "test-workspace").await?;
let encrypted = windmill_common::variables::encrypt(&mc, SECRET_VALUE);
// Runtime-checked query (not the `query!` macro) so no offline `.sqlx` cache
// entry is needed for this test-only insert.
sqlx::query(
"INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
VALUES ('test-workspace', 'f/locked/secret_token', $1, true, 'Locked secret', '{}')",
)
.bind(&encrypted)
.execute(&db)
.await?;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/resources/mcp_tools");
let path = "u/test-user-3/evil_mcp";
// ---- CORE REGRESSION: the developer can read the resource but must NOT be
// able to resolve the locked secret. They are denied (401) at the
// variable-RLS gate, before any MCP connection is attempted, and the
// secret never appears in the response.
let (status, body) = get(&base, path, "SECRET_TOKEN_3").await;
assert_eq!(
status,
reqwest::StatusCode::UNAUTHORIZED,
"developer must be denied resolving a secret they can't read (got {status}): {body}"
);
assert!(
!body.contains(SECRET_VALUE),
"the locked secret must never leak to the developer: {body}"
);
assert!(
body.contains("don't have access"),
"denial should come from the variable-RLS gate, not a connection error: {body}"
);
// Pre-fix, the token was decrypted as admin and the handler proceeded to the
// connection step; that path must no longer be reached for the developer.
assert!(
!body.contains("Failed to connect to MCP server"),
"developer must be blocked before the connection step (would mean the token was resolved): {body}"
);
// ---- NO OVER-BLOCKING: an admin clears the variable-RLS gate, so the token
// resolves and the request only fails later at the connect/SSRF step.
// A different failure mode (not 401, reaches the connection) proves the
// legitimate read still works.
let (status, body) = get(&base, path, "SECRET_TOKEN").await;
assert_ne!(
status,
reqwest::StatusCode::UNAUTHORIZED,
"admin must clear the variable-RLS gate (got {status}): {body}"
);
assert!(
body.contains("Failed to connect to MCP server"),
"admin should resolve the token and only fail at the connect/SSRF step: {body}"
);
Ok(())
}
backend/tests/otel.rs
+120
View File
@@ -507,3 +507,123 @@ async fn test_root_job_span_attributes_values() {
assert_eq!(get_attr("workspace_id"), "test-workspace");
assert_eq!(get_attr("script_path"), "f/test/script");
}
// ═══════════════════════════════════════════════════════════════════════
// INBOUND TRACE CONTEXT (W3C traceparent → span link)
// ═══════════════════════════════════════════════════════════════════════
const SAMPLE_TRACEPARENT: &str = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01";
fn sample_trace_id() -> opentelemetry::trace::TraceId {
opentelemetry::trace::TraceId::from_hex("0af7651916cd43dd8448eb211c80319c").unwrap()
}
fn sample_span_id() -> opentelemetry::trace::SpanId {
opentelemetry::trace::SpanId::from_hex("b7ad6b7169203331").unwrap()
}
#[test]
fn test_span_cx_from_traceparent_valid() {
let cx = span_cx_from_traceparent(SAMPLE_TRACEPARENT).expect("valid traceparent");
assert_eq!(cx.trace_id(), sample_trace_id());
assert_eq!(cx.span_id(), sample_span_id());
assert!(cx.is_remote());
assert!(cx.is_sampled());
}
#[test]
fn test_span_cx_from_traceparent_unsampled_flag() {
let cx = span_cx_from_traceparent("00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-00")
.expect("valid traceparent");
assert!(!cx.is_sampled());
}
#[test]
fn test_span_cx_from_traceparent_malformed() {
for bad in [
"",
"garbage",
"00-tooshort-b7ad6b7169203331-01",
// missing flags field
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331",
// trailing extra field
"00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01-extra",
// all-zero trace id / span id are invalid per the spec
"00-00000000000000000000000000000000-b7ad6b7169203331-01",
"00-0af7651916cd43dd8448eb211c80319c-0000000000000000-01",
// non-hex
"00-zzf7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01",
] {
assert!(
span_cx_from_traceparent(bad).is_none(),
"expected None for {bad:?}"
);
}
}
fn job_with_traceparent(tp: Option<&str>) -> windmill_queue::MiniPulledJob {
let mut job = make_test_job(uuid::Uuid::new_v4(), None);
if let Some(tp) = tp {
let mut args = std::collections::HashMap::new();
args.insert(
windmill_common::jobs::WM_TRACEPARENT.to_string(),
windmill_common::worker::to_raw_value(&tp),
);
job.args = Some(sqlx::types::Json(args));
}
job
}
#[test]
fn test_inbound_span_cx_from_job_present() {
let job = job_with_traceparent(Some(SAMPLE_TRACEPARENT));
let cx = windmill_worker::otel_ee::inbound_span_cx_from_job(&job).expect("link expected");
assert_eq!(cx.trace_id(), sample_trace_id());
assert_eq!(cx.span_id(), sample_span_id());
}
#[test]
fn test_inbound_span_cx_from_job_absent_or_malformed() {
// No reserved key (e.g. a flow step or internally-created job) → no link.
assert!(
windmill_worker::otel_ee::inbound_span_cx_from_job(&job_with_traceparent(None)).is_none()
);
// Malformed header is ignored rather than producing a bogus link.
assert!(
windmill_worker::otel_ee::inbound_span_cx_from_job(&job_with_traceparent(Some("garbage")))
.is_none()
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_root_job_span_relocated_to_inbound_trace() {
let state = ensure_setup().await;
state.span_exporter.reset();
let job = job_with_traceparent(Some(SAMPLE_TRACEPARENT));
let job_id = job.id;
windmill_worker::otel_ee::add_root_flow_job_to_otlp(&job, true);
let spans = state.span_exporter.get_finished_spans().unwrap();
let span = spans
.iter()
.find(|s| s.name == "full_job")
.expect("full_job span not found");
// Relocated into the inbound trace, keeping the job-UUID-derived span id and
// parented on the inbound caller span.
assert_eq!(span.span_context.trace_id(), sample_trace_id());
let expected_span_id =
opentelemetry::trace::SpanId::from_bytes(job_id.as_u64_pair().1.to_be_bytes());
assert_eq!(span.span_context.span_id(), expected_span_id);
assert_eq!(span.parent_span_id, sample_span_id());
// Linked back to the UUID-derived context so trace-by-job-id still resolves.
assert_eq!(span.links.links.len(), 1);
let expected_uuid_trace =
opentelemetry::trace::TraceId::from_bytes(job_id.as_u128().to_be_bytes());
assert_eq!(
span.links.links[0].span_context.trace_id(),
expected_uuid_trace
);
}
backend/tests/preview_native_tag.rs
+122
View File
@@ -0,0 +1,122 @@
/*
* Regression tests for WIN-2007.
*
* Previewing a TypeScript script carrying the `//native` annotation used to be
* pushed with `language = bun` (what the editor sends), so the job was tagged
* `bun` and routed to a regular bun worker. A native-mode worker neither matches
* the `bun` tag nor accepts a non-native `script_lang`, so previewing a `//native`
* script on a native-only worker setup failed even though the *deployed* version
* of the same script runs fine (as `bunnative` / tag `nativets`).
*
* `push` now reconciles the preview language with the `//native` annotation,
* mirroring the deploy-time logic in `worker_lockfiles`. These tests assert the
* queued job ends up with the right `script_lang` and `tag` for every combination
* of declared language and annotation. No worker is spawned — we only inspect the
* row `push` writes.
*/
use sqlx::{Pool, Postgres};
use windmill_common::{
jobs::{JobPayload, RawCode},
scripts::ScriptLang,
};
use windmill_queue::PushIsolationLevel;
async fn push_preview_and_get_row(
db: &Pool<Postgres>,
content: &str,
language: ScriptLang,
) -> (String, Option<ScriptLang>) {
let hm_args = std::collections::HashMap::new();
let job = JobPayload::Code(RawCode {
hash: None,
content: content.to_string(),
path: None,
language,
lock: None,
concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default()
.into(),
debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(),
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
modules: None,
tag: None,
});
let tx = PushIsolationLevel::IsolatedRoot(db.clone());
let (uuid, tx) = windmill_queue::push(
db,
tx,
"test-workspace",
job,
windmill_queue::PushArgs::from(&hm_args),
/* user */ "test-user",
/* email */ "test@windmill.dev",
/* permissioned_as */ "u/test-user".to_string(),
/* token_prefix */ None,
/* scheduled_for */ None,
/* schedule_path */ None,
/* parent_job */ None,
/* root_job */ None,
/* flow_innermost_root_job */ None,
/* job_id */ None,
/* is_flow_step */ false,
/* same_worker */ false,
None,
true,
None,
None,
None,
None,
None,
false,
None,
None,
None,
)
.await
.expect("push must succeed");
tx.commit().await.unwrap();
let row = sqlx::query!(
r#"SELECT tag, script_lang AS "script_lang: ScriptLang" FROM v2_job WHERE id = $1"#,
uuid
)
.fetch_one(db)
.await
.unwrap();
(row.tag, row.script_lang)
}
const NATIVE_CONTENT: &str = r#"//native
export function main(x: number) {
return x;
}
"#;
const PLAIN_CONTENT: &str = r#"export function main(x: number) {
return x;
}
"#;
/// The reported case: editor sends `bun`, content has `//native`. The preview
/// must be promoted to `bunnative` so it tags `nativets` and a native worker
/// (which rejects non-native `script_lang`) can run it.
#[sqlx::test(fixtures("base"))]
async fn test_bun_with_native_annotation_becomes_nativets(db: Pool<Postgres>) {
let (tag, lang) = push_preview_and_get_row(&db, NATIVE_CONTENT, ScriptLang::Bun).await;
assert_eq!(lang, Some(ScriptLang::Bunnative));
assert_eq!(tag, "nativets");
}
/// Guard: a plain bun preview (no `//native`) must stay `bun` / tag `bun`, so
/// the promotion above doesn't broadly retag normal previews.
#[sqlx::test(fixtures("base"))]
async fn test_bun_without_native_annotation_stays_bun(db: Pool<Postgres>) {
let (tag, lang) = push_preview_and_get_row(&db, PLAIN_CONTENT, ScriptLang::Bun).await;
assert_eq!(lang, Some(ScriptLang::Bun));
assert_eq!(tag, "bun");
}
backend/tests/workspace_fairness.rs
+1688
View File
File diff suppressed because it is too large Load Diff
backend/windmill-ai/Cargo.toml
+2 -1
View File
@@ -6,7 +6,7 @@ edition.workspace = true
[features]
default = []
bedrock = ["dep:aws-sdk-bedrockruntime", "dep:aws-credential-types", "dep:aws-smithy-types", "dep:aws-config"]
bedrock = ["dep:aws-sdk-bedrock", "dep:aws-sdk-bedrockruntime", "dep:aws-credential-types", "dep:aws-smithy-types", "dep:aws-config"]
mcp = ["dep:windmill-mcp"]
[lib]
@@ -42,4 +42,5 @@ ulid.workspace = true
aws-config = { workspace = true, optional = true }
aws-credential-types = { workspace = true, optional = true }
aws-smithy-types = { workspace = true, optional = true }
aws-sdk-bedrock = { workspace = true, optional = true }
aws-sdk-bedrockruntime = { workspace = true, optional = true }
backend/windmill-ai/src/ai_bedrock.rs
+36
View File
@@ -754,6 +754,26 @@ pub fn bedrock_stream_event_to_tool_start(
}
}
pub fn bedrock_stream_event_to_tool_start_with_block_index(
event: &ConverseStreamOutput,
) -> Option<(usize, StreamingToolCall)> {
match event {
ConverseStreamOutput::ContentBlockStart(start) => {
let block_index = usize::try_from(start.content_block_index()).ok()?;
let tool_use = start.start().and_then(|s| s.as_tool_use().ok())?;
Some((
block_index,
StreamingToolCall {
id: tool_use.tool_use_id().to_string(),
name: tool_use.name().to_string(),
arguments: String::new(),
},
))
}
_ => None,
}
}
/// Extract tool use input delta from stream
pub fn bedrock_stream_event_to_tool_delta(event: &ConverseStreamOutput) -> Option<String> {
match event {
@@ -765,6 +785,22 @@ pub fn bedrock_stream_event_to_tool_delta(event: &ConverseStreamOutput) -> Optio
}
}
pub fn bedrock_stream_event_to_tool_delta_with_block_index(
event: &ConverseStreamOutput,
) -> Option<(usize, String)> {
match event {
ConverseStreamOutput::ContentBlockDelta(delta) => {
let block_index = usize::try_from(delta.content_block_index()).ok()?;
let input = delta
.delta()
.and_then(|d| d.as_tool_use().ok())
.map(|tool_use| tool_use.input().to_string())?;
Some((block_index, input))
}
_ => None,
}
}
/// Check if stream event indicates content block stop
pub fn bedrock_stream_event_is_block_stop(event: &ConverseStreamOutput) -> bool {
matches!(event, ConverseStreamOutput::ContentBlockStop(_))
backend/windmill-ai/src/ai_providers.rs
+3 -2
View File
@@ -20,13 +20,14 @@ where
lazy_static::lazy_static! {
static ref OPENAI_AZURE_BASE_PATH: Option<String> = std::env::var("OPENAI_AZURE_BASE_PATH").ok();
static ref ALLOW_PRIVATE_AI_BASE_URLS: bool = std::env::var("ALLOW_PRIVATE_AI_BASE_URLS")
pub static ref ALLOW_PRIVATE_AI_BASE_URLS: bool = std::env::var("ALLOW_PRIVATE_AI_BASE_URLS")
.ok()
.map(|v| v == "true" || v == "1")
.unwrap_or(false);
}
pub const OPENAI_BASE_URL: &str = "https://api.openai.com/v1";
pub const DEEPSEEK_BASE_URL: &str = "https://api.deepseek.com/v1";
pub const GOOGLE_AI_BASE_URL: &str = "https://generativelanguage.googleapis.com/v1beta";
/// Empty string signals BedrockClient::from_env() to use the region from AWS environment/config
@@ -106,7 +107,7 @@ impl AIProvider {
Ok(azure_base_path.unwrap_or("https://api.openai.com/v1".to_string()))
}
AIProvider::DeepSeek => Ok("https://api.deepseek.com/v1".to_string()),
AIProvider::DeepSeek => Ok(DEEPSEEK_BASE_URL.to_string()),
AIProvider::GoogleAI => Ok(GOOGLE_AI_BASE_URL.to_string()),
AIProvider::Groq => Ok("https://api.groq.com/openai/v1".to_string()),
AIProvider::OpenRouter => Ok("https://openrouter.ai/api/v1".to_string()),
backend/windmill-ai/src/credentials.rs
+25
View File
@@ -0,0 +1,25 @@
use std::collections::HashMap;
use crate::ai_providers::{AIPlatform, AIProvider};
/// Resolved provider credentials shared by API proxy and worker execution.
///
/// Raw API resources and worker agent payloads convert into this shape at their
/// execution boundaries. Request-specific state such as the selected model stays
/// outside this type.
#[derive(Clone, Debug)]
pub struct ProviderCredentials {
pub provider: AIProvider,
pub base_url: String,
pub api_key: Option<String>,
pub access_token: Option<String>,
pub organization_id: Option<String>,
pub user: Option<String>,
pub region: Option<String>,
pub aws_access_key_id: Option<String>,
pub aws_secret_access_key: Option<String>,
pub aws_session_token: Option<String>,
pub platform: AIPlatform,
pub enable_1m_context: bool,
pub custom_headers: HashMap<String, String>,
}
backend/windmill-ai/src/lib.rs
+1
View File
@@ -4,6 +4,7 @@ pub mod ai_cache;
pub mod ai_google;
pub mod ai_providers;
pub mod ai_types;
pub mod credentials;
pub mod image_handler;
pub mod providers;
pub mod proxy;
backend/windmill-ai/src/providers/anthropic.rs
+1 -2
View File
@@ -729,8 +729,7 @@ impl QueryBuilder for AnthropicQueryBuilder {
mod tests {
use super::*;
use crate::{
proxy::{ProviderCredentials, ProxyBuildArgs},
query_builder::QueryBuilder,
credentials::ProviderCredentials, proxy::ProxyBuildArgs, query_builder::QueryBuilder,
};
use http::{HeaderMap, HeaderValue, Method};
use std::collections::HashMap;
backend/windmill-ai/src/providers/bedrock.rs
+875 -8
View File
@@ -7,21 +7,753 @@
//! - Helper utilities
use crate::{
ai_bedrock::{
bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop,
bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta,
bedrock_stream_event_to_tool_delta_with_block_index, bedrock_stream_event_to_tool_start,
bedrock_stream_event_to_tool_start_with_block_index, build_tool_config,
create_inference_config, format_bedrock_error, openai_messages_to_bedrock,
streaming_tool_calls_to_openai, BearerTokenProvider, BedrockClient, StreamingToolCall,
},
ai_providers::USE_ENV_REGION,
ai_types::{OpenAIFunction, OpenAIToolCall, ToolDefFunction},
image_handler::prepare_messages_for_api,
proxy::ProxyBuildArgs,
query_builder::{ParsedResponse, StreamEventSink},
types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef},
};
use bytes::Bytes;
use futures::{stream::BoxStream, StreamExt};
use http::{HeaderMap, Method, StatusCode};
use serde::Deserialize;
use std::collections::HashMap;
use windmill_common::{client::AuthedClient, error::Error};
// Import shared Bedrock helpers for provider orchestration.
use crate::ai_bedrock::{
bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop,
bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta,
bedrock_stream_event_to_tool_start, build_tool_config, create_inference_config,
format_bedrock_error, openai_messages_to_bedrock, streaming_tool_calls_to_openai,
BedrockClient, StreamingToolCall,
};
// ============================================================================
// Native Proxy Execution
// ============================================================================
/// OpenAI-format request body for Bedrock SDK proxy handlers.
#[derive(Deserialize, Debug)]
struct OpenAIRequest {
messages: Vec<OpenAIMessage>,
#[serde(default)]
tools: Option<Vec<OpenAIToolDef>>,
#[serde(default)]
tool_choice: Option<serde_json::Value>,
#[serde(default)]
max_tokens: Option<i32>,
#[serde(default)]
temperature: Option<f32>,
}
#[derive(Deserialize, Debug)]
struct OpenAIToolDef {
#[serde(default)]
#[allow(dead_code)]
r#type: Option<String>,
function: OpenAIToolFunction,
}
#[derive(Deserialize, Debug)]
struct OpenAIToolFunction {
name: String,
#[serde(default)]
description: Option<String>,
#[serde(default)]
parameters: Option<serde_json::Value>,
}
#[derive(Deserialize, Debug)]
struct BedrockProxyChatRequest {
model: String,
#[serde(default)]
stream: bool,
}
enum BedrockAuthConfig {
BearerToken(String),
IamCredentials {
access_key_id: String,
secret_access_key: String,
session_token: Option<String>,
},
Environment,
}
pub enum BedrockProxyResponseBody {
Fixed(Bytes),
Stream(BoxStream<'static, std::result::Result<Bytes, std::io::Error>>),
}
pub struct BedrockProxyResponse {
pub status_code: StatusCode,
pub headers: HeaderMap,
pub body: BedrockProxyResponseBody,
}
/// Handle a workspace Bedrock proxy request through the AWS SDK.
///
/// The API still owns credential resolution, route authorization, auditing, and
/// cache behavior. This helper owns Bedrock-specific control-plane and
/// OpenAI-compatible Converse transformations.
pub async fn handle_bedrock_proxy(
args: &ProxyBuildArgs<'_>,
) -> Result<BedrockProxyResponse, Error> {
let region = args.credentials.region.as_deref().unwrap_or(USE_ENV_REGION);
if *args.method == Method::GET {
return match args.path {
"foundation-models" => list_foundation_models(args, region).await,
"inference-profiles" => list_inference_profiles(args, region).await,
_ => Err(Error::BadRequest(format!(
"Unsupported AWS Bedrock proxy path: {}",
args.path
))),
};
}
if *args.method != Method::POST {
return Err(Error::BadRequest(format!(
"Unsupported AWS Bedrock proxy method: {}",
args.method
)));
}
let request: BedrockProxyChatRequest = serde_json::from_slice(args.body)
.map_err(|e| Error::internal_err(format!("Failed to parse request body: {}", e)))?;
if request.stream {
handle_bedrock_sdk_streaming(&request.model, args.body, args, region).await
} else {
handle_bedrock_sdk_non_streaming(&request.model, args.body, args, region).await
}
}
fn determine_auth_config(
api_key: Option<&str>,
aws_access_key_id: Option<&str>,
aws_secret_access_key: Option<&str>,
aws_session_token: Option<&str>,
) -> BedrockAuthConfig {
if let Some(key) = api_key.filter(|k| !k.is_empty()) {
BedrockAuthConfig::BearerToken(key.to_string())
} else if let (Some(access_key_id), Some(secret_access_key)) = (
aws_access_key_id.filter(|s| !s.is_empty()),
aws_secret_access_key.filter(|s| !s.is_empty()),
) {
BedrockAuthConfig::IamCredentials {
access_key_id: access_key_id.to_string(),
secret_access_key: secret_access_key.to_string(),
session_token: aws_session_token
.filter(|token| !token.is_empty())
.map(str::to_string),
}
} else {
BedrockAuthConfig::Environment
}
}
async fn create_bedrock_client(
args: &ProxyBuildArgs<'_>,
region: &str,
) -> Result<BedrockClient, Error> {
match determine_auth_config(
args.credentials.api_key.as_deref(),
args.credentials.aws_access_key_id.as_deref(),
args.credentials.aws_secret_access_key.as_deref(),
args.credentials.aws_session_token.as_deref(),
) {
BedrockAuthConfig::BearerToken(key) => BedrockClient::from_bearer_token(key, region).await,
BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key, session_token } => {
BedrockClient::from_credentials(access_key_id, secret_access_key, session_token, region)
.await
}
BedrockAuthConfig::Environment => BedrockClient::from_env(region).await,
}
}
fn build_tool_config_from_request(
tools: Option<&[OpenAIToolDef]>,
tool_choice: Option<&serde_json::Value>,
enable_prompt_caching: bool,
) -> Result<Option<aws_sdk_bedrockruntime::types::ToolConfiguration>, Error> {
if let Some(tools) = tools {
let tool_defs: Vec<ToolDef> = tools
.iter()
.map(|t| ToolDef {
r#type: "function".to_string(),
function: ToolDefFunction {
name: t.function.name.clone(),
description: t.function.description.clone(),
parameters: Box::from(
serde_json::value::RawValue::from_string(
serde_json::to_string(
&t.function
.parameters
.clone()
.unwrap_or(serde_json::json!({})),
)
.unwrap_or_default(),
)
.unwrap_or_else(|_| {
serde_json::value::RawValue::from_string("{}".to_string()).unwrap()
}),
),
},
})
.collect();
let force_tool_use = tool_choice
.map(|tc| tc == "required" || tc.as_str() == Some("required"))
.unwrap_or(false);
build_tool_config(Some(&tool_defs), force_tool_use, enable_prompt_caching)
} else {
Ok(None)
}
}
async fn create_bedrock_control_client(
args: &ProxyBuildArgs<'_>,
region: &str,
) -> Result<aws_sdk_bedrock::Client, Error> {
use aws_config::BehaviorVersion;
let region_provider = aws_sdk_bedrock::config::Region::new(region.to_string());
match determine_auth_config(
args.credentials.api_key.as_deref(),
args.credentials.aws_access_key_id.as_deref(),
args.credentials.aws_secret_access_key.as_deref(),
args.credentials.aws_session_token.as_deref(),
) {
BedrockAuthConfig::BearerToken(key) => {
let config = aws_sdk_bedrock::config::Builder::new()
.region(region_provider)
.behavior_version(BehaviorVersion::latest())
.token_provider(BearerTokenProvider::new(key))
.build();
Ok(aws_sdk_bedrock::Client::from_conf(config))
}
BedrockAuthConfig::IamCredentials { access_key_id, secret_access_key, session_token } => {
let credentials = aws_credential_types::Credentials::new(
access_key_id,
secret_access_key,
session_token,
None,
"windmill",
);
let config = aws_sdk_bedrock::config::Builder::new()
.region(region_provider)
.behavior_version(BehaviorVersion::latest())
.credentials_provider(credentials)
.build();
Ok(aws_sdk_bedrock::Client::from_conf(config))
}
BedrockAuthConfig::Environment => {
let config = aws_config::defaults(BehaviorVersion::latest())
.region(region_provider)
.load()
.await;
Ok(aws_sdk_bedrock::Client::new(&config))
}
}
}
async fn list_foundation_models(
args: &ProxyBuildArgs<'_>,
region: &str,
) -> Result<BedrockProxyResponse, Error> {
let client = create_bedrock_control_client(args, region).await?;
let response = client
.list_foundation_models()
.send()
.await
.map_err(|e| Error::internal_err(format!("Failed to list foundation models: {}", e)))?;
let models: Vec<serde_json::Value> = response
.model_summaries()
.iter()
.map(|m| {
serde_json::json!({
"modelId": m.model_id(),
"modelName": m.model_name(),
"providerName": m.provider_name(),
"modelArn": m.model_arn(),
"inputModalities": m.input_modalities().iter().map(|i| i.as_str()).collect::<Vec<_>>(),
"outputModalities": m.output_modalities().iter().map(|o| o.as_str()).collect::<Vec<_>>(),
"responseStreamingSupported": m.response_streaming_supported(),
"inferenceTypesSupported": m.inference_types_supported().iter().map(|i| i.as_str()).collect::<Vec<_>>(),
})
})
.collect();
let body = serde_json::to_vec(&serde_json::json!({ "modelSummaries": models }))
.map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?;
Ok(BedrockProxyResponse {
status_code: StatusCode::OK,
headers: json_response_headers(),
body: BedrockProxyResponseBody::Fixed(Bytes::from(body)),
})
}
async fn list_inference_profiles(
args: &ProxyBuildArgs<'_>,
region: &str,
) -> Result<BedrockProxyResponse, Error> {
let client = create_bedrock_control_client(args, region).await?;
let response =
client.list_inference_profiles().send().await.map_err(|e| {
Error::internal_err(format!("Failed to list inference profiles: {}", e))
})?;
let profiles: Vec<serde_json::Value> = response
.inference_profile_summaries()
.iter()
.map(|p| {
serde_json::json!({
"inferenceProfileId": p.inference_profile_id(),
"inferenceProfileName": p.inference_profile_name(),
"inferenceProfileArn": p.inference_profile_arn(),
"description": p.description(),
"status": p.status().as_str(),
"type": p.r#type().as_str(),
})
})
.collect();
let body = serde_json::to_vec(&serde_json::json!({ "inferenceProfileSummaries": profiles }))
.map_err(|e| Error::internal_err(format!("Failed to serialize response: {}", e)))?;
Ok(BedrockProxyResponse {
status_code: StatusCode::OK,
headers: json_response_headers(),
body: BedrockProxyResponseBody::Fixed(Bytes::from(body)),
})
}
async fn handle_bedrock_sdk_streaming(
model: &str,
body: &[u8],
args: &ProxyBuildArgs<'_>,
region: &str,
) -> Result<BedrockProxyResponse, Error> {
let openai_req: OpenAIRequest = serde_json::from_slice(body)
.map_err(|e| Error::internal_err(format!("Failed to parse OpenAI request: {}", e)))?;
let bedrock_client = create_bedrock_client(args, region).await?;
let enable_prompt_caching = bedrock_model_supports_prompt_caching(model);
let (bedrock_messages, system_prompts) =
openai_messages_to_bedrock(&openai_req.messages, enable_prompt_caching)?;
let inference_config = create_inference_config(openai_req.temperature, openai_req.max_tokens);
let tool_config = build_tool_config_from_request(
openai_req.tools.as_deref(),
openai_req.tool_choice.as_ref(),
enable_prompt_caching,
)?;
let mut request_builder = bedrock_client
.client()
.converse_stream()
.model_id(model)
.set_messages(Some(bedrock_messages));
if !system_prompts.is_empty() {
request_builder = request_builder.set_system(Some(system_prompts));
}
if let Some(config) = inference_config {
request_builder = request_builder.inference_config(config);
}
if let Some(config) = tool_config {
request_builder = request_builder.set_tool_config(Some(config));
}
tracing::debug!("Bedrock SDK streaming: sending converse_stream request");
let stream_output = request_builder.send().await.map_err(|e| {
let error_msg = format!("Bedrock SDK streaming error: {}", format_bedrock_error(&e));
tracing::error!("Bedrock SDK streaming failed: {}", error_msg);
Error::internal_err(error_msg)
})?;
tracing::debug!("Bedrock SDK streaming: stream established successfully");
Ok(BedrockProxyResponse {
status_code: StatusCode::OK,
headers: event_stream_response_headers(),
body: BedrockProxyResponseBody::Stream(
sdk_stream_to_sse(stream_output.stream, model.to_string()).boxed(),
),
})
}
pub fn sdk_stream_to_sse(
stream: aws_sdk_bedrockruntime::primitives::event_stream::EventReceiver<
aws_sdk_bedrockruntime::types::ConverseStreamOutput,
aws_sdk_bedrockruntime::types::error::ConverseStreamOutputError,
>,
model: String,
) -> impl futures::Stream<Item = std::result::Result<Bytes, std::io::Error>> + Send {
let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple());
let created = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
async_stream::stream! {
let mut stream = stream;
let mut state = BedrockSseStreamState::new(id, model, created);
loop {
match stream.recv().await {
Ok(Some(event)) => {
for chunk in bedrock_sse_chunks_for_event(&event, &mut state) {
yield Ok(chunk);
}
}
Ok(None) => break,
Err(e) => {
yield Err(std::io::Error::new(
std::io::ErrorKind::Other,
e.to_string(),
));
break;
}
}
}
yield Ok(Bytes::from("data: [DONE]\n\n"));
}
}
#[derive(Debug)]
struct BedrockSseStreamState {
id: String,
model: String,
created: u64,
tool_calls: HashMap<usize, (String, String, String)>,
tool_block_indexes: HashMap<usize, usize>,
next_tool_index: usize,
}
impl BedrockSseStreamState {
fn new(id: String, model: String, created: u64) -> Self {
Self {
id,
model,
created,
tool_calls: HashMap::new(),
tool_block_indexes: HashMap::new(),
next_tool_index: 0,
}
}
}
fn bedrock_sse_chunks_for_event(
event: &aws_sdk_bedrockruntime::types::ConverseStreamOutput,
state: &mut BedrockSseStreamState,
) -> Vec<Bytes> {
let mut chunks = Vec::new();
if let Some((block_index, tool_call)) =
bedrock_stream_event_to_tool_start_with_block_index(event)
{
let index = state.next_tool_index;
state.next_tool_index += 1;
state.tool_block_indexes.insert(block_index, index);
state.tool_calls.insert(
index,
(tool_call.id.clone(), tool_call.name.clone(), String::new()),
);
let chunk = serde_json::json!({
"id": state.id,
"object": "chat.completion.chunk",
"created": state.created,
"model": state.model,
"choices": [{
"index": 0,
"delta": {
"tool_calls": [{
"index": index,
"id": tool_call.id,
"type": "function",
"function": {
"name": tool_call.name,
"arguments": ""
}
}]
},
"finish_reason": serde_json::Value::Null
}]
});
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
if let Some(text) = bedrock_stream_event_to_text(event) {
let chunk = serde_json::json!({
"id": state.id,
"object": "chat.completion.chunk",
"created": state.created,
"model": state.model,
"choices": [{
"index": 0,
"delta": {
"content": text
},
"finish_reason": serde_json::Value::Null
}]
});
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
if let Some((block_index, input_delta)) =
bedrock_stream_event_to_tool_delta_with_block_index(event)
{
if let Some(index) = state.tool_block_indexes.get(&block_index).copied() {
if let Some((_id, _name, ref mut args)) = state.tool_calls.get_mut(&index) {
args.push_str(&input_delta);
let chunk = serde_json::json!({
"id": state.id,
"object": "chat.completion.chunk",
"created": state.created,
"model": state.model,
"choices": [{
"index": 0,
"delta": {
"tool_calls": [{
"index": index,
"function": {
"arguments": input_delta
}
}]
},
"finish_reason": serde_json::Value::Null
}]
});
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
}
}
if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(stop) = event {
let stop_reason = stop.stop_reason().as_str();
let finish_reason = match stop_reason {
"end_turn" => "stop",
"max_tokens" => "length",
"tool_use" => "tool_calls",
"stop_sequence" => "stop",
"guardrail_intervened" | "content_filtered" => "content_filter",
_ => "stop",
};
let chunk = serde_json::json!({
"id": state.id,
"object": "chat.completion.chunk",
"created": state.created,
"model": state.model,
"choices": [{
"index": 0,
"delta": {},
"finish_reason": finish_reason
}]
});
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
chunks
}
async fn handle_bedrock_sdk_non_streaming(
model: &str,
body: &[u8],
args: &ProxyBuildArgs<'_>,
region: &str,
) -> Result<BedrockProxyResponse, Error> {
let openai_req: OpenAIRequest = serde_json::from_slice(body)
.map_err(|e| Error::internal_err(format!("Failed to parse OpenAI request: {}", e)))?;
let bedrock_client = create_bedrock_client(args, region).await?;
let enable_prompt_caching = bedrock_model_supports_prompt_caching(model);
let (bedrock_messages, system_prompts) =
openai_messages_to_bedrock(&openai_req.messages, enable_prompt_caching)?;
let inference_config = create_inference_config(openai_req.temperature, openai_req.max_tokens);
let tool_config = build_tool_config_from_request(
openai_req.tools.as_deref(),
openai_req.tool_choice.as_ref(),
enable_prompt_caching,
)?;
let mut request_builder = bedrock_client
.client()
.converse()
.model_id(model)
.set_messages(Some(bedrock_messages));
if !system_prompts.is_empty() {
request_builder = request_builder.set_system(Some(system_prompts));
}
if let Some(config) = inference_config {
request_builder = request_builder.inference_config(config);
}
if let Some(config) = tool_config {
request_builder = request_builder.set_tool_config(Some(config));
}
tracing::debug!("Bedrock SDK non-streaming: sending converse request");
let response = request_builder.send().await.map_err(|e| {
let error_msg = format!(
"Bedrock SDK non-streaming error: {}",
format_bedrock_error(&e)
);
tracing::error!("Bedrock SDK non-streaming failed: {}", error_msg);
Error::internal_err(error_msg)
})?;
tracing::debug!(
"Bedrock SDK non-streaming: response received, stop_reason={}",
response.stop_reason().as_str()
);
let id = format!("chatcmpl-{}", uuid::Uuid::new_v4().simple());
let created = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_secs();
let stop_reason = response.stop_reason().as_str();
let finish_reason = match stop_reason {
"end_turn" => "stop",
"max_tokens" => "length",
"tool_use" => "tool_calls",
"stop_sequence" => "stop",
"guardrail_intervened" | "content_filtered" => "content_filter",
_ => "stop",
};
let mut text_content = String::new();
let mut tool_calls: Vec<OpenAIToolCall> = Vec::new();
if let Some(aws_sdk_bedrockruntime::types::ConverseOutput::Message(message)) = response.output()
{
for block in message.content() {
match block {
aws_sdk_bedrockruntime::types::ContentBlock::Text(text) => {
text_content.push_str(text);
}
aws_sdk_bedrockruntime::types::ContentBlock::ToolUse(tool_use) => {
let input_json = document_to_json(tool_use.input());
tool_calls.push(OpenAIToolCall {
id: tool_use.tool_use_id().to_string(),
function: OpenAIFunction {
name: tool_use.name().to_string(),
arguments: serde_json::to_string(&input_json).unwrap_or_default(),
},
r#type: "function".to_string(),
extra_content: None,
});
}
_ => {}
}
}
}
let message = if !tool_calls.is_empty() {
serde_json::json!({
"role": "assistant",
"content": if text_content.is_empty() { serde_json::Value::Null } else { serde_json::Value::String(text_content) },
"tool_calls": tool_calls
})
} else {
serde_json::json!({
"role": "assistant",
"content": text_content
})
};
let usage = if let Some(usage_data) = response.usage() {
serde_json::json!({
"prompt_tokens": usage_data.input_tokens(),
"completion_tokens": usage_data.output_tokens(),
"total_tokens": usage_data.total_tokens()
})
} else {
serde_json::json!({
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0
})
};
let openai_resp = serde_json::json!({
"id": id,
"object": "chat.completion",
"created": created,
"model": model,
"choices": [{
"index": 0,
"message": message,
"finish_reason": finish_reason
}],
"usage": usage
});
let body = serde_json::to_vec(&openai_resp)
.map_err(|e| Error::internal_err(format!("Failed to serialize OpenAI response: {}", e)))?;
Ok(BedrockProxyResponse {
status_code: StatusCode::OK,
headers: json_response_headers(),
body: BedrockProxyResponseBody::Fixed(Bytes::from(body)),
})
}
fn json_response_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert("content-type", "application/json".parse().unwrap());
headers
}
fn event_stream_response_headers() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.insert("content-type", "text/event-stream".parse().unwrap());
headers.insert("cache-control", "no-cache".parse().unwrap());
headers.insert("connection", "keep-alive".parse().unwrap());
headers
}
fn document_to_json(doc: &aws_smithy_types::Document) -> serde_json::Value {
match doc {
aws_smithy_types::Document::Object(map) => {
let mut json_map = serde_json::Map::new();
for (key, value) in map {
json_map.insert(key.clone(), document_to_json(value));
}
serde_json::Value::Object(json_map)
}
aws_smithy_types::Document::Array(values) => {
serde_json::Value::Array(values.iter().map(document_to_json).collect())
}
aws_smithy_types::Document::Number(number) => match number {
aws_smithy_types::Number::PosInt(number) => serde_json::Value::Number((*number).into()),
aws_smithy_types::Number::NegInt(number) => serde_json::Value::Number((*number).into()),
aws_smithy_types::Number::Float(number) => serde_json::json!(*number),
},
aws_smithy_types::Document::String(value) => serde_json::Value::String(value.clone()),
aws_smithy_types::Document::Bool(value) => serde_json::Value::Bool(*value),
aws_smithy_types::Document::Null => serde_json::Value::Null,
}
}
// ============================================================================
// Query Builder
@@ -256,3 +988,138 @@ impl BedrockQueryBuilder {
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use aws_sdk_bedrockruntime::types::{
ContentBlockDelta, ContentBlockDeltaEvent, ContentBlockStart, ContentBlockStartEvent,
ContentBlockStopEvent, ConverseStreamOutput, ToolUseBlockDelta, ToolUseBlockStart,
};
fn sse_json(chunk: &Bytes) -> serde_json::Value {
let chunk = std::str::from_utf8(chunk).expect("SSE chunk should be UTF-8");
let payload = chunk
.strip_prefix("data: ")
.and_then(|chunk| chunk.strip_suffix("\n\n"))
.expect("chunk should be SSE data");
serde_json::from_str(payload).expect("chunk should contain JSON")
}
#[test]
fn determine_auth_config_prioritizes_bearer_token() {
let config = determine_auth_config(
Some("bearer-token"),
Some("AKIA123"),
Some("secret"),
Some("session-token"),
);
match config {
BedrockAuthConfig::BearerToken(token) => assert_eq!(token, "bearer-token"),
_ => panic!("expected bearer token auth config"),
}
}
#[test]
fn determine_auth_config_uses_iam_with_optional_session_token() {
let config =
determine_auth_config(None, Some("AKIA123"), Some("secret"), Some("session-token"));
match config {
BedrockAuthConfig::IamCredentials {
access_key_id,
secret_access_key,
session_token,
} => {
assert_eq!(access_key_id, "AKIA123");
assert_eq!(secret_access_key, "secret");
assert_eq!(session_token.as_deref(), Some("session-token"));
}
_ => panic!("expected IAM auth config"),
}
}
#[test]
fn determine_auth_config_treats_empty_session_token_as_none() {
let config = determine_auth_config(None, Some("AKIA123"), Some("secret"), Some(""));
match config {
BedrockAuthConfig::IamCredentials { session_token, .. } => {
assert!(session_token.is_none());
}
_ => panic!("expected IAM auth config"),
}
}
#[test]
fn determine_auth_config_falls_back_to_environment() {
let config = determine_auth_config(None, Some("AKIA123"), None, Some("session-token"));
assert!(matches!(config, BedrockAuthConfig::Environment));
}
#[test]
fn bedrock_sse_tool_indexes_ignore_text_block_stops() {
let mut state =
BedrockSseStreamState::new("chatcmpl-test".to_string(), "model".to_string(), 1);
let text_delta = ConverseStreamOutput::ContentBlockDelta(
ContentBlockDeltaEvent::builder()
.content_block_index(0)
.delta(ContentBlockDelta::Text("hello".to_string()))
.build()
.unwrap(),
);
assert_eq!(
bedrock_sse_chunks_for_event(&text_delta, &mut state).len(),
1
);
let text_stop = ConverseStreamOutput::ContentBlockStop(
ContentBlockStopEvent::builder()
.content_block_index(0)
.build()
.unwrap(),
);
assert!(bedrock_sse_chunks_for_event(&text_stop, &mut state).is_empty());
let tool_start = ConverseStreamOutput::ContentBlockStart(
ContentBlockStartEvent::builder()
.content_block_index(1)
.start(ContentBlockStart::ToolUse(
ToolUseBlockStart::builder()
.tool_use_id("call_1")
.name("lookup")
.build()
.unwrap(),
))
.build()
.unwrap(),
);
let start_chunks = bedrock_sse_chunks_for_event(&tool_start, &mut state);
let start_json = sse_json(&start_chunks[0]);
assert_eq!(
start_json["choices"][0]["delta"]["tool_calls"][0]["index"],
0
);
let tool_delta = ConverseStreamOutput::ContentBlockDelta(
ContentBlockDeltaEvent::builder()
.content_block_index(1)
.delta(ContentBlockDelta::ToolUse(
ToolUseBlockDelta::builder()
.input("{\"city\":\"Paris\"}")
.build()
.unwrap(),
))
.build()
.unwrap(),
);
let delta_chunks = bedrock_sse_chunks_for_event(&tool_delta, &mut state);
let delta_json = sse_json(&delta_chunks[0]);
assert_eq!(
delta_json["choices"][0]["delta"]["tool_calls"][0]["index"],
0
);
}
}
backend/windmill-ai/src/providers/google_ai.rs
+33 -1
View File
@@ -408,6 +408,8 @@ fn build_google_ai_model_endpoint(
action: &str,
is_vertex: bool,
) -> String {
let model = model.strip_prefix("models/").unwrap_or(model);
if is_vertex {
format!("{}/{}:{}", base_url, model, action)
} else {
@@ -416,6 +418,10 @@ fn build_google_ai_model_endpoint(
}
fn add_google_ai_auth_header(headers: &mut Vec<(String, String)>, api_key: &str, is_vertex: bool) {
// Native Google AI proxy intentionally does not apply AI_HTTP_HEADERS or
// resource custom headers yet. Gemini/Vertex header semantics are
// provider-specific; keep this limited to required auth headers until
// explicit custom-header support is designed.
if is_vertex {
headers.push(("Authorization".to_string(), format!("Bearer {}", api_key)));
} else {
@@ -685,7 +691,7 @@ impl QueryBuilder for GoogleAIQueryBuilder {
#[cfg(test)]
mod tests {
use super::*;
use crate::{ai_providers::AIProvider, proxy::ProviderCredentials};
use crate::{ai_providers::AIProvider, credentials::ProviderCredentials};
use std::collections::HashMap;
fn credentials(base_url: &str, platform: AIPlatform) -> ProviderCredentials {
@@ -749,6 +755,32 @@ mod tests {
assert!(body["contents"].is_array());
}
#[test]
fn builds_standard_google_ai_endpoint_from_model_resource_name() {
assert_eq!(
build_google_ai_model_endpoint(
"https://generativelanguage.googleapis.com/v1beta",
"models/gemini-2.0-flash",
"generateContent",
false,
),
"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent"
);
}
#[test]
fn builds_vertex_google_ai_endpoint_from_model_resource_name() {
assert_eq!(
build_google_ai_model_endpoint(
"https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/publishers/google/models",
"models/gemini-2.0-flash",
"streamGenerateContent",
true,
),
"https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/publishers/google/models/gemini-2.0-flash:streamGenerateContent"
);
}
#[test]
fn builds_vertex_google_ai_streaming_proxy_request() {
let credentials = credentials(
backend/windmill-ai/src/providers/mod.rs
+3 -21
View File
@@ -7,8 +7,7 @@ pub mod openrouter;
pub mod other;
use crate::{
ai_providers::AIProvider, proxy::ProviderCredentials, query_builder::QueryBuilder,
types::ProviderWithResource,
ai_providers::AIProvider, credentials::ProviderCredentials, query_builder::QueryBuilder,
};
use self::{
@@ -16,25 +15,8 @@ use self::{
openrouter::OpenRouterQueryBuilder, other::OtherQueryBuilder,
};
/// Factory function to create the appropriate query builder for a provider.
pub fn create_query_builder(provider: &ProviderWithResource) -> Box<dyn QueryBuilder> {
match provider.kind {
AIProvider::GoogleAI => {
Box::new(GoogleAIQueryBuilder::new(provider.get_platform().clone()))
}
AIProvider::OpenAI => Box::new(OpenAIQueryBuilder::new(provider.kind.clone())),
AIProvider::Anthropic => Box::new(AnthropicQueryBuilder::new(
provider.kind.clone(),
provider.get_platform().clone(),
provider.get_enable_1m_context(),
)),
AIProvider::OpenRouter => Box::new(OpenRouterQueryBuilder::new()),
_ => Box::new(OtherQueryBuilder::new(provider.kind.clone())),
}
}
/// Factory function to create the appropriate query builder from resolved proxy credentials.
pub fn create_proxy_query_builder(credentials: &ProviderCredentials) -> Box<dyn QueryBuilder> {
/// Factory function to create the appropriate query builder from resolved credentials.
pub fn create_query_builder(credentials: &ProviderCredentials) -> Box<dyn QueryBuilder> {
match credentials.provider {
AIProvider::GoogleAI => Box::new(GoogleAIQueryBuilder::new(credentials.platform.clone())),
AIProvider::OpenAI => Box::new(OpenAIQueryBuilder::new(credentials.provider.clone())),
backend/windmill-ai/src/proxy.rs
+6 -21
View File
@@ -4,29 +4,11 @@ use http::{HeaderMap, Method};
use serde_json::value::RawValue;
use windmill_common::error::{Error, Result};
use crate::ai_providers::{AIPlatform, AIProvider};
use crate::ai_providers::AIProvider;
use crate::credentials::ProviderCredentials;
use crate::utils::AI_HTTP_HEADERS;
/// Resolved provider credentials and proxy-specific context.
///
/// This is intentionally separate from the worker's `ProviderWithResource`: API
/// proxy credentials are already resolved from workspace or instance resources.
#[derive(Clone, Debug)]
pub struct ProviderCredentials {
pub provider: AIProvider,
pub base_url: String,
pub api_key: Option<String>,
pub access_token: Option<String>,
pub organization_id: Option<String>,
pub user: Option<String>,
pub region: Option<String>,
pub aws_access_key_id: Option<String>,
pub aws_secret_access_key: Option<String>,
pub aws_session_token: Option<String>,
pub platform: AIPlatform,
pub enable_1m_context: bool,
pub custom_headers: HashMap<String, String>,
}
pub mod fim;
/// Inputs needed to transform an OpenAI-compatible proxy request for a provider.
pub struct ProxyBuildArgs<'a> {
@@ -166,6 +148,9 @@ pub(crate) fn add_user_to_body(body: &[u8], user: &str) -> Result<Vec<u8>> {
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
use crate::ai_providers::AIPlatform;
fn credentials(provider: AIProvider, base_url: &str) -> ProviderCredentials {
ProviderCredentials {
backend/windmill-ai/src/proxy/fim.rs
+212
View File
@@ -0,0 +1,212 @@
use bytes::Bytes;
use serde::Deserialize;
use serde_json::json;
use windmill_common::error::{Error, Result};
use crate::ai_providers::{AIProvider, DEEPSEEK_BASE_URL};
#[derive(Debug, Eq, PartialEq)]
pub struct FimProxyTransform {
pub body: Bytes,
pub path: String,
pub base_url: Option<String>,
}
#[derive(Deserialize)]
struct FimRequest {
model: String,
prompt: String,
suffix: Option<String>,
temperature: Option<f32>,
max_tokens: Option<u32>,
stop: Option<Vec<String>>,
}
pub fn supports_native_fim(provider: &AIProvider) -> bool {
matches!(provider, AIProvider::Mistral | AIProvider::DeepSeek)
}
fn deepseek_fim_base_url(base_url: &str) -> String {
let trimmed = base_url.trim_end_matches('/');
let deepseek_root_base_url = DEEPSEEK_BASE_URL
.strip_suffix("/v1")
.unwrap_or(DEEPSEEK_BASE_URL);
if trimmed == DEEPSEEK_BASE_URL || trimmed == deepseek_root_base_url {
return format!("{deepseek_root_base_url}/beta");
}
if let Some(prefix) = trimmed.strip_suffix("/v1") {
return format!("{prefix}/beta");
}
trimmed.to_string()
}
pub fn maybe_transform_fim_request(
provider: &AIProvider,
path: &str,
base_url: &str,
body: &[u8],
) -> Result<Option<FimProxyTransform>> {
if !path.contains("fim/completions") {
return Ok(None);
}
if matches!(provider, AIProvider::DeepSeek) {
return Ok(Some(FimProxyTransform {
body: Bytes::copy_from_slice(body),
path: "completions".to_string(),
base_url: Some(deepseek_fim_base_url(base_url)),
}));
}
if !supports_native_fim(provider) {
return transform_fim_to_chat_completions(body).map(Some);
}
Ok(None)
}
fn transform_fim_to_chat_completions(body: &[u8]) -> Result<FimProxyTransform> {
let fim_req: FimRequest = serde_json::from_slice(body)
.map_err(|e| Error::BadRequest(format!("Failed to parse FIM request: {}", e)))?;
let suffix = fim_req.suffix.unwrap_or_default();
let system_prompt = "You are a code completion assistant. Complete the code at the <CURSOR/> position between the given prefix and suffix. Output ONLY the code that goes at the cursor - no explanations, no markdown, no repeating the prefix or suffix.";
let user_content = format!(
"<PREFIX>\n{}\n<CURSOR/>\n<SUFFIX>\n{}",
fim_req.prompt, suffix
);
let chat_req = json!({
"model": fim_req.model,
"messages": [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_content}
],
"temperature": fim_req.temperature.unwrap_or(0.0),
"max_tokens": fim_req.max_tokens.unwrap_or(256),
"stop": fim_req.stop
});
let body = serde_json::to_vec(&chat_req)
.map_err(|e| Error::internal_err(format!("Failed to serialize chat request: {}", e)))?;
Ok(FimProxyTransform {
body: Bytes::from(body),
path: "chat/completions".to_string(),
base_url: None,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mistral_keeps_native_fim_request() {
let transformed = maybe_transform_fim_request(
&AIProvider::Mistral,
"fim/completions",
"https://api.mistral.ai/v1",
br#"{}"#,
)
.unwrap();
assert!(transformed.is_none());
assert!(supports_native_fim(&AIProvider::Mistral));
assert!(supports_native_fim(&AIProvider::DeepSeek));
assert!(!supports_native_fim(&AIProvider::OpenAI));
}
#[test]
fn deepseek_fim_base_url_uses_beta_endpoint() {
assert_eq!(
deepseek_fim_base_url("https://api.deepseek.com/v1"),
"https://api.deepseek.com/beta"
);
assert_eq!(
deepseek_fim_base_url("https://api.deepseek.com/v1/"),
"https://api.deepseek.com/beta"
);
assert_eq!(
deepseek_fim_base_url("https://api.deepseek.com"),
"https://api.deepseek.com/beta"
);
assert_eq!(
deepseek_fim_base_url("https://proxy.example/deepseek/v1"),
"https://proxy.example/deepseek/beta"
);
assert_eq!(
deepseek_fim_base_url("https://proxy.example/deepseek/beta"),
"https://proxy.example/deepseek/beta"
);
}
#[test]
fn deepseek_fim_request_uses_beta_completions_endpoint() {
let body = br#"{"model":"deepseek-v4-pro","prompt":"return ","suffix":";"}"#;
let transformed = maybe_transform_fim_request(
&AIProvider::DeepSeek,
"fim/completions",
DEEPSEEK_BASE_URL,
body,
)
.unwrap()
.expect("DeepSeek FIM should be routed to the beta completions endpoint");
assert_eq!(transformed.path, "completions");
assert_eq!(
transformed.base_url.as_deref(),
Some("https://api.deepseek.com/beta")
);
assert_eq!(transformed.body, Bytes::copy_from_slice(body));
}
#[test]
fn openai_fim_request_is_transformed_to_chat_completion() {
let transformed = maybe_transform_fim_request(
&AIProvider::OpenAI,
"fim/completions",
"https://api.openai.com/v1",
br#"{
"model": "gpt-4.1",
"prompt": "fn main() {",
"suffix": "}",
"stop": ["\n\n"]
}"#,
)
.unwrap()
.expect("OpenAI FIM should be transformed");
assert_eq!(transformed.path, "chat/completions");
assert_eq!(transformed.base_url, None);
let body: serde_json::Value = serde_json::from_slice(&transformed.body).unwrap();
assert_eq!(body["model"], "gpt-4.1");
assert_eq!(body["temperature"], 0.0);
assert_eq!(body["max_tokens"], 256);
assert_eq!(body["stop"], serde_json::json!(["\n\n"]));
assert_eq!(body["messages"][1]["role"], "user");
assert_eq!(
body["messages"][1]["content"],
"<PREFIX>\nfn main() {\n<CURSOR/>\n<SUFFIX>\n}"
);
}
#[test]
fn invalid_fim_body_is_bad_request() {
let err = maybe_transform_fim_request(
&AIProvider::OpenAI,
"fim/completions",
"https://api.openai.com/v1",
br#"{"model": 1}"#,
)
.unwrap_err();
assert!(matches!(err, Error::BadRequest(_)));
}
}
backend/windmill-ai/src/types.rs
+29
View File
@@ -18,6 +18,7 @@ pub struct McpToolSource {
use crate::{
ai_google::sanitize_schema_for_google,
ai_providers::{empty_string_as_none, AIProvider},
credentials::ProviderCredentials,
};
use windmill_common::{db::DB, error::Error, flow_status::AgentAction, flows::FlowModule};
use windmill_parser::Typ;
@@ -222,6 +223,34 @@ impl ProviderWithResource {
.await
}
/// Convert worker agent provider input into resolved runtime credentials.
///
/// Callers must only pass resources that were already authorized for the
/// current job/workspace; this helper does not perform access checks.
pub async fn to_provider_credentials(&self, db: &DB) -> Result<ProviderCredentials, Error> {
let base_url = if self.kind == AIProvider::AWSBedrock {
String::new()
} else {
self.get_base_url(db).await?
};
Ok(ProviderCredentials {
provider: self.kind.clone(),
base_url,
api_key: self.resource.api_key.clone(),
access_token: None,
organization_id: None,
user: None,
region: self.resource.region.clone(),
aws_access_key_id: self.resource.aws_access_key_id.clone(),
aws_secret_access_key: self.resource.aws_secret_access_key.clone(),
aws_session_token: self.resource.aws_session_token.clone(),
platform: self.resource.platform.clone(),
enable_1m_context: self.resource.enable_1m_context,
custom_headers: self.resource.headers.clone(),
})
}
#[cfg(feature = "bedrock")]
pub fn get_region(&self) -> Option<&str> {
self.resource.region.as_deref()
backend/windmill-api-auth/src/lib.rs
+583
View File
@@ -235,6 +235,246 @@ where
Ok(())
}
/// Returns the caller's "real" scope restrictions: every scope other than
/// `if_jobs:filter_tags:` tag filters. `None` means the token is unscoped and
/// has the full privileges of its user; `Some` means it is restricted to the
/// returned scopes. An empty or filter-tags-only scope list is treated as
/// unscoped, mirroring `check_scopes`/`check_route_access`.
fn scope_restrictions(scopes: Option<&[String]>) -> Option<Vec<&String>> {
let restrictions: Vec<&String> = scopes?
.iter()
.filter(|s| !s.starts_with("if_jobs:filter_tags:"))
.collect();
(!restrictions.is_empty()).then_some(restrictions)
}
/// Enforce monotonic privilege when a token lifecycle endpoint mints or rescopes
/// a credential on behalf of `authed`: the resulting credential must never be
/// more privileged than the caller's own token.
///
/// - An unscoped caller may grant any scopes (this is the existing UI/CLI flow).
/// - A scope-restricted caller may only grant scopes that are a subset of its
/// own, and may never produce an unscoped credential.
///
/// Without this, a `users:write` token could create or rescope a token to be
/// unscoped, and a `users:read` token could refresh into an unscoped session —
/// escaping its own restrictions.
pub fn ensure_scopes_within_caller(
authed: &ApiAuthed,
requested_scopes: Option<&[String]>,
) -> error::Result<()> {
if let Some(caller_restrictions) = scope_restrictions(authed.scopes.as_deref()) {
let Some(requested_restrictions) = scope_restrictions(requested_scopes) else {
return Err(Error::PermissionDenied(
"A scope-restricted token cannot create or update a token with broader (unscoped) \
privileges"
.to_string(),
));
};
// MCP scopes (`mcp:all`, `mcp:favorites`, `mcp:scripts:*`, etc.) use a
// custom format that ScopeDefinition::from_scope_string parses
// permissively but the MCP runtime interprets via its own parser
// (parse_mcp_scopes). The two views disagree — e.g. the generic parser
// accepts `mcp:scripts` as an unrestricted-resource scope, while the
// MCP runtime ignores it as unrecognized but interprets `mcp:scripts:*`
// as granting all scripts. So generic containment would silently allow
// `mcp:scripts` → `mcp:scripts:*` (a widening). Legitimate MCP token
// issuance goes through the OAuth gateway (mcp/oauth_server.rs), not
// these user-token endpoints, so require byte-identical match for MCP
// scopes here rather than trying to mirror MCP semantics in two places.
// Unparseable non-MCP caller scopes are intentionally dropped
// (fail-closed): a caller scope that fails to parse can only narrow
// the set of requested scopes that get covered, never widen it.
// Unparseable requested scopes surface as `BadRequest`, which is what
// we want — the client is sending garbage.
let parsed_caller: Vec<ScopeDefinition> = caller_restrictions
.iter()
.filter(|s| !s.starts_with("mcp:"))
.filter_map(|s| ScopeDefinition::from_scope_string(s).ok())
.collect();
let caller_mcp: std::collections::HashSet<&str> = caller_restrictions
.iter()
.filter(|s| s.starts_with("mcp:"))
.map(|s| s.as_str())
.collect();
for requested in requested_restrictions {
if requested.starts_with("mcp:") {
if !caller_mcp.contains(requested.as_str()) {
return Err(Error::PermissionDenied(format!(
"A scope-restricted token cannot grant MCP scope '{requested}' unless the \
caller holds the same scope verbatim"
)));
}
continue;
}
let requested_scope = ScopeDefinition::from_scope_string(requested)?;
let covered = parsed_caller
.iter()
.any(|caller_scope| scope_contains(caller_scope, &requested_scope));
if !covered {
return Err(Error::PermissionDenied(format!(
"A scope-restricted token cannot grant scope '{requested}' which exceeds its \
own scopes"
)));
}
}
}
// `if_jobs:filter_tags:` fences which job tags a token can run on (enforced
// at job operations as `v2_job.tag = ANY(...)`), and is checked independently
// of domain/action/resource subset. A caller restricted by filter_tags must
// not be able to mint or rescope a credential that drops or widens the fence
// — even if the caller has no other scope restrictions (filter_tags-only
// tokens otherwise look "unscoped" to `scope_restrictions`).
if let Some(caller_tags) = first_filter_tags(authed.scopes.as_deref()) {
let Some(requested_tags) = first_filter_tags(requested_scopes) else {
return Err(Error::PermissionDenied(
"A token restricted by if_jobs:filter_tags cannot mint or rescope a token that \
drops the tag restriction"
.to_string(),
));
};
let caller_set: std::collections::HashSet<&str> = caller_tags.iter().copied().collect();
for tag in &requested_tags {
if !caller_set.contains(tag) {
return Err(Error::PermissionDenied(format!(
"A token restricted by if_jobs:filter_tags cannot grant tag '{tag}' which is \
not within its own filter_tags"
)));
}
}
}
Ok(())
}
/// Tags from the first `if_jobs:filter_tags:<a,b,...>` scope, matching the
/// semantics of [`get_scope_tags`] (which is what the job runtime consults).
/// Returns `None` if no such scope is present.
fn first_filter_tags(scopes: Option<&[String]>) -> Option<Vec<&str>> {
scopes?.iter().find_map(|s| {
s.strip_prefix("if_jobs:filter_tags:")
.map(|tags| tags.split(',').collect())
})
}
/// Whether `caller` grants at least everything `requested` grants (directional
/// containment).
///
/// This is intentionally NOT `ScopeDefinition::includes`: that method answers
/// "does this scope grant access to a required action" using OR semantics over
/// resources (any overlap counts, and a `*` on either side matches), which is
/// correct for access checks but unsafe for subset checks — it would let a
/// token scoped to `scripts:read:f/team/a` mint `scripts:read:*` or
/// `scripts:read:f/team/a,f/other/b`. Subset containment instead requires that
/// EVERY requested resource is covered by SOME caller resource.
fn scope_contains(caller: &ScopeDefinition, requested: &ScopeDefinition) -> bool {
if caller.domain != requested.domain {
return false;
}
// write subsumes read; otherwise the action must match exactly.
match (caller.action.as_str(), requested.action.as_str()) {
(c, r) if c == r || (c == "write" && r == "read") => {}
_ => return false,
}
if caller.domain == "jobs" && caller.action == "run" {
match (&caller.kind, &requested.kind) {
(Some(caller_kind), Some(requested_kind)) if caller_kind != requested_kind => {
return false
}
// Caller pinned to a kind, but the request covers any kind.
(Some(_), None) => return false,
_ => {}
}
}
match (&caller.resource, &requested.resource) {
// Caller is unrestricted on resources: covers everything.
(None, _) => true,
// Caller is resource-restricted but the request is not: broader.
(Some(_), None) => false,
(Some(caller_resources), Some(requested_resources)) => {
resource_set_contains(caller_resources, requested_resources)
}
}
}
/// Every resource in `requested` must be covered by some resource in `caller`.
fn resource_set_contains(caller: &[String], requested: &[String]) -> bool {
if caller.iter().any(|r| r == "*") {
return true;
}
requested
.iter()
.all(|req| req != "*" && caller.iter().any(|c| resource_covers(c, req)))
}
/// Directional: does the single caller resource pattern cover `requested`?
/// `caller` may be an exact path or a `<prefix>/*` subtree wildcard; `requested`
/// may itself be a subtree wildcard, in which case the whole requested subtree
/// must fall within the caller's subtree.
fn resource_covers(caller: &str, requested: &str) -> bool {
if caller == requested {
return true;
}
let Some(prefix) = caller.strip_suffix("/*") else {
// An exact caller resource only covers itself (handled above).
return false;
};
let requested_base = requested.strip_suffix("/*").unwrap_or(requested);
requested_base == prefix
|| (requested_base.starts_with(prefix)
&& requested_base.as_bytes().get(prefix.len()) == Some(&b'/'))
}
/// Returns a predicate that checks whether `path` is within the token's
/// scope for `{domain}:{action}:{path}`. For tokens without scope
/// restrictions (no scopes at all, or only `if_jobs:filter_tags:*` scopes),
/// the predicate always returns `true`.
///
/// Pre-parses the token's scopes once so the returned closure can cheaply
/// filter large listings without re-parsing on each call.
pub fn build_scope_path_predicate(
authed: &ApiAuthed,
domain: &str,
action: &str,
) -> impl Fn(&str) -> bool {
// Mirror check_scopes semantics: a token is "scope-restricted" iff it has
// at least one non-`if_jobs:filter_tags:` scope. Unparseable scopes still
// count as restrictive — they just match nothing.
let (is_scoped_token, parsed): (bool, Vec<ScopeDefinition>) = match authed.scopes.as_ref() {
Some(scopes) => {
let mut is_scoped = false;
let parsed = scopes
.iter()
.filter(|s| !s.starts_with("if_jobs:filter_tags:"))
.inspect(|_| is_scoped = true)
.filter_map(|s| ScopeDefinition::from_scope_string(s).ok())
.collect();
(is_scoped, parsed)
}
None => (false, Vec::new()),
};
let domain = domain.to_string();
let action = action.to_string();
move |path: &str| -> bool {
if !is_scoped_token {
return true;
}
let required =
match ScopeDefinition::from_scope_string(&format!("{}:{}:{}", domain, action, path)) {
Ok(r) => r,
Err(_) => return false,
};
parsed.iter().any(|s| s.includes(&required))
}
}
pub async fn require_devops_role(db: &DB, email: &str) -> error::Result<()> {
let is_devops = is_devops_email(db, email).await?;
@@ -530,6 +770,13 @@ impl NewToken {
}
}
/// Low-level token mint shared by trusted callers (the user-facing
/// `tokens/create` handler and internal mints such as native-trigger webhook
/// tokens). It does NOT enforce that `token_config.scopes` is within the
/// caller's own scopes — callers exposed to untrusted input must call
/// [`ensure_scopes_within_caller`] first (internal narrowing mints intentionally
/// skip it, since their scopes derive from the action being authorized, not the
/// caller's token).
pub async fn create_token_internal(
tx: &mut sqlx::PgConnection,
db: &DB,
@@ -803,3 +1050,339 @@ pub fn require_path_read_access_for_preview(
))),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn authed_with_scopes(scopes: Option<Vec<&str>>) -> ApiAuthed {
ApiAuthed {
scopes: scopes.map(|v| v.into_iter().map(String::from).collect()),
..Default::default()
}
}
#[test]
fn predicate_no_scopes_allows_all() {
let authed = authed_with_scopes(None);
let allowed = build_scope_path_predicate(&authed, "resources", "read");
assert!(allowed("u/alice/anything"));
assert!(allowed("u/bob/other"));
}
#[test]
fn predicate_tag_filter_only_allows_all() {
let authed = authed_with_scopes(Some(vec!["if_jobs:filter_tags:default"]));
let allowed = build_scope_path_predicate(&authed, "resources", "read");
assert!(allowed("u/alice/foo"));
}
#[test]
fn predicate_single_resource_scope_filters_others() {
// Regression test for WIN-1981: a token scoped to one resource must
// not match unrelated paths in listings (e.g. /resources/list_search).
let authed = authed_with_scopes(Some(vec!["resources:read:u/alice/allowed_resource"]));
let allowed = build_scope_path_predicate(&authed, "resources", "read");
assert!(allowed("u/alice/allowed_resource"));
assert!(!allowed("u/alice/other_resource"));
assert!(!allowed("u/bob/foo"));
}
#[test]
fn predicate_wildcard_scope_matches_subtree() {
let authed = authed_with_scopes(Some(vec!["resources:read:f/team/*"]));
let allowed = build_scope_path_predicate(&authed, "resources", "read");
assert!(allowed("f/team/db"));
assert!(allowed("f/team/sub/nested"));
assert!(!allowed("f/other/db"));
}
#[test]
fn predicate_wrong_domain_is_rejected() {
let authed = authed_with_scopes(Some(vec!["variables:read:u/alice/secret"]));
let allowed = build_scope_path_predicate(&authed, "resources", "read");
assert!(!allowed("u/alice/secret"));
}
#[test]
fn predicate_write_implies_read() {
let authed = authed_with_scopes(Some(vec!["resources:write:u/alice/foo"]));
let allowed = build_scope_path_predicate(&authed, "resources", "read");
assert!(allowed("u/alice/foo"));
assert!(!allowed("u/alice/bar"));
}
fn opt_scopes(scopes: Option<Vec<&str>>) -> Option<Vec<String>> {
scopes.map(|v| v.into_iter().map(String::from).collect())
}
// Regression tests for WIN-1999: scoped user tokens must not be able to
// mint or rescope credentials with broader privileges than themselves.
#[test]
fn unscoped_caller_can_grant_anything() {
let authed = authed_with_scopes(None);
assert!(ensure_scopes_within_caller(&authed, None).is_ok());
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["jobs:run:scripts"])).as_deref()
)
.is_ok());
}
#[test]
fn filter_tags_only_caller_is_unrestricted_on_domain_action_dimension() {
// The domain/action/resource subset check treats filter-tags-only as
// unrestricted, mirroring check_scopes/check_route_access. The tag
// dimension is checked separately (see filter_tags_dimension_is_monotonic).
let authed = authed_with_scopes(Some(vec!["if_jobs:filter_tags:default"]));
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["users:write", "if_jobs:filter_tags:default"])).as_deref()
)
.is_ok());
}
#[test]
fn filter_tags_dimension_is_monotonic() {
// Caller restricted to tag fence "a" cannot drop the fence …
let single = authed_with_scopes(Some(vec!["if_jobs:filter_tags:a"]));
assert!(ensure_scopes_within_caller(&single, None).is_err());
assert!(
ensure_scopes_within_caller(&single, opt_scopes(Some(vec!["users:read"])).as_deref())
.is_err(),
"minting a token without filter_tags must be rejected"
);
// … cannot widen to a tag it lacks …
assert!(ensure_scopes_within_caller(
&single,
opt_scopes(Some(vec!["if_jobs:filter_tags:a,b"])).as_deref()
)
.is_err());
// … and cannot mint a token fenced on a disjoint tag.
assert!(ensure_scopes_within_caller(
&single,
opt_scopes(Some(vec!["if_jobs:filter_tags:b"])).as_deref()
)
.is_err());
// Narrowing or matching the tag fence is allowed.
let multi = authed_with_scopes(Some(vec!["if_jobs:filter_tags:a,b"]));
assert!(ensure_scopes_within_caller(
&multi,
opt_scopes(Some(vec!["if_jobs:filter_tags:a"])).as_deref()
)
.is_ok());
assert!(ensure_scopes_within_caller(
&multi,
opt_scopes(Some(vec!["if_jobs:filter_tags:a,b"])).as_deref()
)
.is_ok());
// A caller with a real scope plus a tag fence cannot drop just the fence.
let mixed = authed_with_scopes(Some(vec!["jobs:run:scripts", "if_jobs:filter_tags:a"]));
assert!(ensure_scopes_within_caller(
&mixed,
opt_scopes(Some(vec!["jobs:run:scripts"])).as_deref()
)
.is_err());
assert!(ensure_scopes_within_caller(
&mixed,
opt_scopes(Some(vec!["jobs:run:scripts", "if_jobs:filter_tags:a"])).as_deref()
)
.is_ok());
// An unrestricted caller may grant filter_tags freely.
let unscoped = authed_with_scopes(None);
assert!(ensure_scopes_within_caller(
&unscoped,
opt_scopes(Some(vec!["if_jobs:filter_tags:x"])).as_deref()
)
.is_ok());
}
#[test]
fn scoped_caller_cannot_mint_unscoped_token() {
// Primitive 2 in the report: a users:write token minting an unscoped token.
let authed = authed_with_scopes(Some(vec!["users:write"]));
assert!(ensure_scopes_within_caller(&authed, None).is_err());
// Empty scope list is effectively unscoped and must also be rejected.
assert!(ensure_scopes_within_caller(&authed, Some(&[])).is_err());
// A scope list of only tag filters is effectively unscoped too.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["if_jobs:filter_tags:default"])).as_deref()
)
.is_err());
}
#[test]
fn scoped_caller_cannot_remove_its_own_scopes() {
// Primitive 3 in the report: a users:write token setting its scopes to null.
let authed = authed_with_scopes(Some(vec!["users:write"]));
assert!(ensure_scopes_within_caller(&authed, None).is_err());
}
#[test]
fn scoped_caller_cannot_grant_scope_it_lacks() {
let authed = authed_with_scopes(Some(vec!["users:write"]));
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["jobs:run:scripts"])).as_deref()
)
.is_err());
}
#[test]
fn scoped_caller_can_grant_subset_of_own_scopes() {
let authed = authed_with_scopes(Some(vec!["users:write", "jobs:run:scripts"]));
// Equal scope is allowed.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["jobs:run:scripts"])).as_deref()
)
.is_ok());
// write implies read, so a narrower read scope is allowed.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["users:read"])).as_deref()
)
.is_ok());
// Tag filters narrow further and are always permitted.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["users:read", "if_jobs:filter_tags:default"])).as_deref()
)
.is_ok());
}
#[test]
fn scoped_caller_cannot_broaden_resource_scope() {
let authed = authed_with_scopes(Some(vec!["scripts:read:f/team/*"]));
// Narrower resource within the subtree is allowed.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["scripts:read:f/team/sub"])).as_deref()
)
.is_ok());
// A nested subtree within the caller's subtree is allowed.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["scripts:read:f/team/sub/*"])).as_deref()
)
.is_ok());
// The subtree root itself is allowed.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["scripts:read:f/team"])).as_deref()
)
.is_ok());
// A path outside the subtree is rejected.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["scripts:read:f/other/x"])).as_deref()
)
.is_err());
// read caller cannot grant write.
assert!(ensure_scopes_within_caller(
&authed,
opt_scopes(Some(vec!["scripts:write:f/team/db"])).as_deref()
)
.is_err());
}
#[test]
fn mcp_scopes_require_byte_identical_match() {
// Regression for the access-grant-OR vs runtime-MCP-parser confusion:
// ScopeDefinition treats `mcp:scripts` as an unrestricted-resource scope
// and `mcp:scripts:*` as a strictly narrower one, so generic containment
// would silently allow widening. The MCP runtime however ignores
// `mcp:scripts` (unrecognized) while `mcp:scripts:*` grants all scripts.
// Legitimate MCP token issuance is the OAuth gateway, not these
// user-token endpoints, so MCP scopes must match the caller verbatim.
// The bypass the reviewer flagged: malformed `mcp:scripts` would widen
// into the real `mcp:scripts:*` under generic containment.
let bypass = authed_with_scopes(Some(vec!["users:write", "mcp:scripts"]));
assert!(ensure_scopes_within_caller(
&bypass,
opt_scopes(Some(vec!["users:write", "mcp:scripts:*"])).as_deref()
)
.is_err());
// A caller without any MCP scope cannot grant one (widening on the MCP
// dimension), even if the rest of the requested scopes are within reach.
let no_mcp = authed_with_scopes(Some(vec!["users:write"]));
assert!(ensure_scopes_within_caller(
&no_mcp,
opt_scopes(Some(vec!["users:write", "mcp:scripts:*"])).as_deref()
)
.is_err());
// Byte-identical MCP scope passes; an additional non-matching MCP scope
// alongside it does not.
let mcp_caller = authed_with_scopes(Some(vec!["mcp:scripts:*"]));
assert!(ensure_scopes_within_caller(
&mcp_caller,
opt_scopes(Some(vec!["mcp:scripts:*"])).as_deref()
)
.is_ok());
assert!(ensure_scopes_within_caller(
&mcp_caller,
opt_scopes(Some(vec!["mcp:scripts:*", "mcp:flows:*"])).as_deref()
)
.is_err());
// Even a narrowing within MCP semantics (`mcp:all` → `mcp:scripts:*`)
// is rejected by the byte-identical rule. This is intentional — these
// endpoints are not the legitimate path for narrowing MCP tokens.
let mcp_all = authed_with_scopes(Some(vec!["mcp:all"]));
assert!(ensure_scopes_within_caller(
&mcp_all,
opt_scopes(Some(vec!["mcp:scripts:*"])).as_deref()
)
.is_err());
}
#[test]
fn scoped_caller_cannot_escalate_to_wildcard_or_superset() {
// Regression for the access-grant-OR vs subset-containment confusion:
// ScopeDefinition::includes would (incorrectly) allow all of these.
let star = authed_with_scopes(Some(vec!["scripts:read:f/team/a"]));
// Minting `*` from a single-path scope must be rejected.
assert!(ensure_scopes_within_caller(
&star,
opt_scopes(Some(vec!["scripts:read:*"])).as_deref()
)
.is_err());
// Minting a broader subtree must be rejected.
assert!(ensure_scopes_within_caller(
&star,
opt_scopes(Some(vec!["scripts:read:f/team/*"])).as_deref()
)
.is_err());
// A comma-separated list that adds an uncovered resource must be rejected,
// even though one element overlaps the caller's scope.
let list = authed_with_scopes(Some(vec!["scripts:read:f/team/a"]));
assert!(ensure_scopes_within_caller(
&list,
opt_scopes(Some(vec!["scripts:read:f/team/a,f/other/b"])).as_deref()
)
.is_err());
// A subset of a multi-resource caller scope is allowed.
let multi = authed_with_scopes(Some(vec!["scripts:read:f/team/a,f/team/b"]));
assert!(ensure_scopes_within_caller(
&multi,
opt_scopes(Some(vec!["scripts:read:f/team/a"])).as_deref()
)
.is_ok());
// A wildcard caller covers any subset, but not `*`-less escalation rules apply
// only when the caller itself lacks `*`.
let wildcard = authed_with_scopes(Some(vec!["scripts:read:*"]));
assert!(ensure_scopes_within_caller(
&wildcard,
opt_scopes(Some(vec!["scripts:read:f/team/a"])).as_deref()
)
.is_ok());
}
}
backend/windmill-api-flows/src/flows.rs
+33 -17
View File
@@ -16,7 +16,8 @@ use axum::{
};
use windmill_api_auth::{
auth::{list_tokens_internal, TruncatedTokenWithEmail},
check_scopes, maybe_refresh_folders, require_owner_of_path, ApiAuthed,
build_scope_path_predicate, check_scopes, maybe_refresh_folders, require_owner_of_path,
ApiAuthed,
};
use windmill_common::workspaces::{check_deploy_rules, RuleCheckResult};
use windmill_common::{
@@ -108,9 +109,10 @@ async fn list_search_flows(
let n = 3;
let mut tx = user_db.begin(&authed).await?;
let allowed = build_scope_path_predicate(&authed, "flows", "read");
let rows = sqlx::query_as::<_, SearchFlow>(
"SELECT flow.path, flow_version.value
FROM flow
FROM flow
LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.workspace_id = $1 LIMIT $2",
)
@@ -119,6 +121,7 @@ async fn list_search_flows(
.fetch_all(&mut *tx)
.await?
.into_iter()
.filter(|r| allowed(&r.path))
.collect::<Vec<_>>();
tx.commit().await?;
Ok(Json(rows))
@@ -212,9 +215,13 @@ async fn list_flows(
let sql = sqlb.sql().map_err(|e| Error::internal_err(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let allowed = build_scope_path_predicate(&authed, "flows", "read");
let rows = sqlx::query_as::<_, ListableFlow>(&sql)
.fetch_all(&mut *tx)
.await?;
.await?
.into_iter()
.filter(|r| allowed(&r.path))
.collect::<Vec<_>>();
tx.commit().await?;
Ok(Json(rows))
}
@@ -558,13 +565,17 @@ async fn create_flow(
w_id
).execute(&mut *tx).await?;
sqlx::query!(
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
nf.path,
&w_id
)
.execute(&mut *tx)
.await?;
// CLI / git-sync deploys ask us to preserve any existing user draft at this
// path instead of wiping it as part of the deploy.
if !nf.skip_draft_deletion.unwrap_or(false) {
sqlx::query!(
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
nf.path,
&w_id
)
.execute(&mut *tx)
.await?;
}
audit_log(
&mut *tx,
@@ -1157,13 +1168,17 @@ async fn update_flow(
})?;
}
sqlx::query!(
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
flow_path,
&w_id
)
.execute(&mut *tx)
.await?;
// CLI / git-sync deploys ask us to preserve any existing user draft at this
// path instead of wiping it as part of the deploy.
if !nf.skip_draft_deletion.unwrap_or(false) {
sqlx::query!(
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'flow'",
flow_path,
&w_id
)
.execute(&mut *tx)
.await?;
}
audit_log(
&mut *tx,
@@ -2031,6 +2046,7 @@ mod tests {
})),
preprocessor_module: None,
same_worker: false,
preserve_step_tags: false,
skip_expr: None,
cache_ttl: None,
cache_ignore_s3_path: None,
backend/windmill-api-groups/src/granular_acls.rs
+3 -1
View File
@@ -383,6 +383,8 @@ async fn remove_granular_acl(
// workspace export.
let table = if kind == "raw_app" { "app" } else { kind };
// SAFETY: `kind` has been validated against the `KINDS` allowlist before reaching this function.
// LIMIT 1: `script` shares (workspace_id, path) across versions, so `old` can
// return >1 row, which would break the scalar subquery in RETURNING.
let obj_o = sqlx::query_scalar::<_, bool>(&format!(
"WITH old AS (
SELECT extra_perms->$1 as old_write FROM {table}
@@ -390,7 +392,7 @@ async fn remove_granular_acl(
)
UPDATE {table} SET extra_perms = extra_perms - $1
WHERE {identifier} = $2 AND workspace_id = $3 AND extra_perms ? $1
RETURNING (SELECT old_write FROM old)::bool"
RETURNING (SELECT old_write FROM old LIMIT 1)::bool"
))
.bind(&owner)
.bind(path)
backend/windmill-api-integration-tests/Cargo.toml
+3
View File
@@ -35,6 +35,9 @@ anyhow.workspace = true
uuid.workspace = true
futures.workspace = true
rand.workspace = true
hmac.workspace = true
sha2.workspace = true
hex.workspace = true
rumqttc.workspace = true
rdkafka.workspace = true
async-nats.workspace = true
backend/windmill-api-integration-tests/tests/fixtures/resource_cache_rls.sql
Vendored
+23
View File
@@ -0,0 +1,23 @@
-- Fixture for the resource-value interpolation cache RLS regression test.
-- Extends base.sql (which defines test-user [admin], test-user-2, test-user-3
-- and their tokens).
--
-- A folder `secret` is readable ONLY by test-user-2 (via extra_perms). It holds a
-- variable and a resource that interpolates it. test-user-3 has no access to the
-- folder, so a cache entry warmed by test-user-2 with allow_cache=true must never
-- be served back to test-user-3.
INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by)
VALUES ('test-workspace', 'secret', 'Secret Folder', '{}',
'{"u/test-user-2": true}', 'test-user');
-- A (non-secret) variable gated to the `secret` folder; its value gets interpolated
-- into the resource value below and ends up in the cached, already-resolved blob.
INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
VALUES ('test-workspace', 'f/secret/db_password', 'LEAKED_FOLDER_SECRET', false,
'Folder-gated secret', '{}');
INSERT INTO resource (workspace_id, path, value, description, resource_type, extra_perms, created_by)
VALUES ('test-workspace', 'f/secret/cache_target',
'{"host": "db.internal", "password": "$var:f/secret/db_password"}',
'Folder-gated resource referencing a folder-gated variable', 'object', '{}', 'test-user');
backend/windmill-api-integration-tests/tests/fixtures/variable_cache_rls.sql
Vendored
+15
View File
@@ -0,0 +1,15 @@
-- Fixture for the variable-value cache RLS regression test.
-- Extends base.sql (which defines test-user [admin], test-user-2, test-user-3
-- and their tokens).
--
-- A folder `secret` is readable ONLY by test-user-2 (via extra_perms). It holds a
-- variable that test-user-2 can read but test-user-3 cannot. A cache entry warmed
-- by test-user-2 with allow_cache=true must never be served back to test-user-3.
INSERT INTO folder (workspace_id, name, display_name, owners, extra_perms, created_by)
VALUES ('test-workspace', 'secret', 'Secret Folder', '{}',
'{"u/test-user-2": true}', 'test-user');
INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms)
VALUES ('test-workspace', 'f/secret/cache_target_var', 'LEAKED_VAR_SECRET', false,
'Folder-gated variable', '{}');
backend/windmill-api-integration-tests/tests/flows.rs
+103 -12
View File
@@ -259,12 +259,10 @@ async fn test_flow_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
// ===== Hub endpoints (require external network, expect 500 or 200) =====
// --- hub/list ---
let resp = authed(client().get(format!(
"http://localhost:{port}/api/flows/hub/list"
)))
.send()
.await
.unwrap();
let resp = authed(client().get(format!("http://localhost:{port}/api/flows/hub/list")))
.send()
.await
.unwrap();
assert!(
resp.status() == 200 || resp.status() == 500,
"hub/list: unexpected status {}",
@@ -272,12 +270,10 @@ async fn test_flow_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
);
// --- hub/get ---
let resp = authed(client().get(format!(
"http://localhost:{port}/api/flows/hub/get/1"
)))
.send()
.await
.unwrap();
let resp = authed(client().get(format!("http://localhost:{port}/api/flows/hub/get/1")))
.send()
.await
.unwrap();
assert!(
resp.status() == 200 || resp.status() == 500,
"hub/get: unexpected status {}",
@@ -286,3 +282,98 @@ async fn test_flow_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
Ok(())
}
/// Regression test for GHSA-2ppx-66jv-wpw5: a path-scoped token must only see
/// the flows within its scope when listing, even though the route-level scope
/// check only validates `domain:action`. Before the fix, `list_search` returned
/// `path` + the full flow `value` for every flow the underlying user could see,
/// leaking out-of-scope flow definitions to narrowly-scoped tokens.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_list_search_scope_filtering(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/flows");
// Create two folders and one flow in each, as the (super-admin) test user.
for folder in ["allowed", "private"] {
let resp = authed(client().post(format!(
"http://localhost:{port}/api/w/test-workspace/folders/create"
)))
.json(&json!({ "name": folder }))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "create folder: {}", resp.text().await?);
}
for path in ["f/allowed/foo", "f/private/bar"] {
let resp = authed(client().post(format!("{base}/create")))
.json(&new_flow(path, "summary"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 201, "create {path}: {}", resp.text().await?);
}
// Helper: GET /list_search with an arbitrary bearer token, returning the set
// of flow paths visible to that token.
async fn list_search_paths(port: u16, token: &str) -> Vec<String> {
let resp = client()
.get(format!(
"http://localhost:{port}/api/w/test-workspace/flows/list_search"
))
.header("Authorization", format!("Bearer {token}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
resp.json::<Vec<serde_json::Value>>()
.await
.unwrap()
.into_iter()
.map(|s| s["path"].as_str().unwrap().to_string())
.collect()
}
// Insert three tokens for the same super-admin user, differing only by scope.
sqlx::query(
"INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES
(encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', 'test@windmill.dev', 'scoped', true, ARRAY['flows:read:f/allowed/*']),
(encode(sha256('BROAD_TOKEN'::bytea), 'hex'), 'BROAD_TOK', 'BROAD_TOKEN', 'test@windmill.dev', 'broad', true, ARRAY['flows:read']),
(encode(sha256('TAG_TOKEN'::bytea), 'hex'), 'TAG_TOK', 'TAG_TOKEN', 'test@windmill.dev', 'tag-only', true, ARRAY['if_jobs:filter_tags:default'])",
)
.execute(&db)
.await?;
// Path-scoped token: only sees flows within `f/allowed/*`.
let scoped = list_search_paths(port, "SCOPED_TOKEN").await;
assert!(
scoped.contains(&"f/allowed/foo".to_string()),
"scoped token should see f/allowed/foo, got: {scoped:?}"
);
assert!(
!scoped.contains(&"f/private/bar".to_string()),
"scoped token must NOT see f/private/bar, got: {scoped:?}"
);
// Broad `flows:read` token: still sees every RLS-visible flow.
let broad = list_search_paths(port, "BROAD_TOKEN").await;
assert!(broad.contains(&"f/allowed/foo".to_string()));
assert!(
broad.contains(&"f/private/bar".to_string()),
"broad flows:read token should see all flows, got: {broad:?}"
);
// Tag-filter-only token is not scope-restricted: unchanged, sees all.
let tag_only = list_search_paths(port, "TAG_TOKEN").await;
assert!(tag_only.contains(&"f/allowed/foo".to_string()));
assert!(tag_only.contains(&"f/private/bar".to_string()));
// Unscoped token (no scopes column set): unchanged, sees all.
let unscoped = list_search_paths(port, "SECRET_TOKEN").await;
assert!(unscoped.contains(&"f/allowed/foo".to_string()));
assert!(unscoped.contains(&"f/private/bar".to_string()));
Ok(())
}
backend/windmill-api-integration-tests/tests/native_triggers.rs
+22 -2
View File
@@ -17,8 +17,8 @@ use windmill_native_triggers::{
decrypt_oauth_data, delete_native_trigger, delete_workspace_integration,
get_workspace_integration,
google::{parse_stop_channel_params, should_renew_channel},
store_native_trigger, store_workspace_integration, NativeTriggerConfig, OAuthConfig,
ServiceName,
require_native_integration_use, store_native_trigger, store_workspace_integration,
NativeTriggerConfig, OAuthConfig, ServiceName,
};
// ============================================================================
@@ -329,6 +329,26 @@ async fn test_token_update_persists(db: Pool<Postgres>) -> anyhow::Result<()> {
// 3. Channel Expiration Renewal — should_renew_channel
// ============================================================================
#[test]
fn test_require_native_integration_use_blocks_operators() {
// Regression: the integration *use* routes (calendar/drive/repo/event pickers)
// must reject read-only operators, who cannot create native triggers and so
// must not be able to drive the admin-configured integration's upstream API.
let mut operator = test_authed();
operator.is_admin = false;
operator.is_operator = true;
assert!(require_native_integration_use(&operator).is_err());
// A regular non-admin author (the population that configures triggers) is allowed.
let mut author = test_authed();
author.is_admin = false;
author.is_operator = false;
assert!(require_native_integration_use(&author).is_ok());
// Admins are allowed.
assert!(require_native_integration_use(&test_authed()).is_ok());
}
#[test]
fn test_should_renew_drive_channel_expired() {
let config = json!({
backend/windmill-api-integration-tests/tests/resources.rs
+111
View File
@@ -477,6 +477,117 @@ async fn test_resource_endpoints(db: Pool<Postgres>) -> anyhow::Result<()> {
Ok(())
}
/// Regression test: the resource-value interpolation cache
/// (`get_value_interpolated?allow_cache=true`) must be identity-scoped. test-user-2
/// (folder access) warms the cache; test-user-3 (no access) must then be denied rather
/// than served the cached, already-decrypted value. Pre-fix the unscoped key returned
/// a 200 with the secret here.
#[sqlx::test(migrations = "../migrations", fixtures("base", "resource_cache_rls"))]
async fn test_resource_value_cache_is_identity_scoped(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let url = format!(
"{}?allow_cache=true",
resource_url(port, "get_value_interpolated", "f/secret/cache_target")
);
let get = |token: &str| {
client()
.get(url.as_str())
.header("Authorization", format!("Bearer {token}"))
};
// test-user-2 has folder access and WARMS the cache.
let resp = get("SECRET_TOKEN_2").send().await?;
assert_eq!(resp.status(), 200);
assert!(resp.text().await?.contains("LEAKED_FOLDER_SECRET"));
// test-user-3 has no folder access: must miss the cache and be denied (401), not leak.
let resp = get("SECRET_TOKEN_3").send().await?;
assert_eq!(resp.status(), 401);
assert!(!resp.text().await?.contains("LEAKED_FOLDER_SECRET"));
Ok(())
}
/// A resource whose value contains a `$WM_*` contextual variable (e.g. `$WM_TOKEN`) is
/// job-dependent and must NEVER be cached — even when first read WITHOUT a `job_id`, where the
/// placeholder is left unresolved (caching that would serve a stale placeholder to a later job
/// read). Any other value — plain, or a non-`$WM_` `$`-string like `$HOME` (which is NOT
/// interpolated, so it's constant) — is job-independent and IS cached, with the entry shared
/// across job contexts (a read carrying a `job_id` still hits it, keeping the hit ratio up).
/// We prove all three by warming each (no job_id), deleting the row directly (cache survives),
/// then re-reading: the job-independent ones are still served from cache — even under a
/// `job_id` — while the `$WM_*` one was never cached and 404s.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_resource_cache_handles_job_context(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/resources");
let plain = "u/test-user/plain_res";
let dollar = "u/test-user/dollar_res"; // non-$WM_ `$`-string: not interpolated, cacheable
let jobctx = "u/test-user/jobctx_res";
for (path, value) in [
(plain, json!({"v": 1})),
(dollar, json!({"d": "$HOME"})),
(jobctx, json!({"j": "$WM_JOB_ID"})),
] {
let resp = authed(client().post(format!("{base}/create")))
.json(
&json!({ "path": path, "value": value, "description": "", "resource_type": "object" }),
)
.send()
.await?;
assert_eq!(resp.status(), 201);
}
let get = |path: &str, query: &str| {
let url = format!("{base}/get_value_interpolated/{path}?{query}");
async move { authed(client().get(url)).send().await.unwrap() }
};
// Warm all three WITHOUT a job context (the placeholder is left unresolved for `jobctx`).
for path in [plain, dollar, jobctx] {
assert_eq!(get(path, "allow_cache=true").await.status(), 200);
}
// Delete the rows directly — bypasses the API/NOTIFY, so the in-memory cache survives.
for path in [plain, dollar, jobctx] {
sqlx::query("DELETE FROM resource WHERE workspace_id = 'test-workspace' AND path = $1")
.bind(path)
.execute(&db)
.await?;
}
// Job-independent values are cached and still served even under a job_id (a random uuid is
// fine: a cache hit short-circuits before any job lookup). `$HOME` is a non-`$WM_` string,
// so it's not interpolated and stays cacheable.
for path in [plain, dollar] {
let resp = get(
path,
"allow_cache=true&job_id=11111111-1111-4111-8111-111111111111",
)
.await;
assert_eq!(
resp.status(),
200,
"job-independent resource ({path}) must stay cached and be served under a job_id"
);
}
// The `$WM_*` resource was never cached → the (now deleted) row is not found.
let resp = get(jobctx, "allow_cache=true").await;
assert_ne!(
resp.status(),
200,
"resource with a $WM_* contextual variable must not be cached"
);
Ok(())
}
#[cfg(feature = "mcp")]
#[sqlx::test(migrations = "../migrations", fixtures("base", "resources_test"))]
async fn test_mcp_tools(db: Pool<Postgres>) -> anyhow::Result<()> {
backend/windmill-api-integration-tests/tests/scripts.rs
+104
View File
@@ -463,3 +463,107 @@ async fn test_auto_parent_resolves_parent_hash(db: Pool<Postgres>) -> anyhow::Re
Ok(())
}
/// Regression test for GHSA-2ppx-66jv-wpw5: a path-scoped token must only see
/// the scripts within its scope when listing, even though the route-level scope
/// check only validates `domain:action`. Before the fix, `list_search` (and
/// `list`) returned `path` + full `content` for every script the underlying
/// user could see, leaking out-of-scope script source to narrowly-scoped tokens.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_list_search_scope_filtering(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/scripts");
// Create two folders and one script in each, as the (super-admin) test user.
for folder in ["allowed", "private"] {
let resp = authed(client().post(format!(
"http://localhost:{port}/api/w/test-workspace/folders/create"
)))
.json(&json!({ "name": folder }))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200, "create folder: {}", resp.text().await?);
}
for (path, content) in [
(
"f/allowed/foo",
"export async function main() { return 'allowed'; }",
),
(
"f/private/bar",
"export async function main() { return 'secret'; }",
),
] {
let resp = authed(client().post(format!("{base}/create")))
.json(&new_script(path, "summary", content))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 201, "create {path}: {}", resp.text().await?);
}
// Helper: GET /list_search with an arbitrary bearer token, returning the set
// of script paths visible to that token.
async fn list_search_paths(port: u16, token: &str) -> Vec<String> {
let resp = client()
.get(format!(
"http://localhost:{port}/api/w/test-workspace/scripts/list_search"
))
.header("Authorization", format!("Bearer {token}"))
.send()
.await
.unwrap();
assert_eq!(resp.status(), 200);
resp.json::<Vec<serde_json::Value>>()
.await
.unwrap()
.into_iter()
.map(|s| s["path"].as_str().unwrap().to_string())
.collect()
}
// Insert three tokens for the same super-admin user, differing only by scope.
sqlx::query(
"INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES
(encode(sha256('SCOPED_TOKEN'::bytea), 'hex'), 'SCOPED_TOK', 'SCOPED_TOKEN', 'test@windmill.dev', 'scoped', true, ARRAY['scripts:read:f/allowed/*']),
(encode(sha256('BROAD_TOKEN'::bytea), 'hex'), 'BROAD_TOK', 'BROAD_TOKEN', 'test@windmill.dev', 'broad', true, ARRAY['scripts:read']),
(encode(sha256('TAG_TOKEN'::bytea), 'hex'), 'TAG_TOK', 'TAG_TOKEN', 'test@windmill.dev', 'tag-only', true, ARRAY['if_jobs:filter_tags:default'])",
)
.execute(&db)
.await?;
// Path-scoped token: only sees scripts within `f/allowed/*`.
let scoped = list_search_paths(port, "SCOPED_TOKEN").await;
assert!(
scoped.contains(&"f/allowed/foo".to_string()),
"scoped token should see f/allowed/foo, got: {scoped:?}"
);
assert!(
!scoped.contains(&"f/private/bar".to_string()),
"scoped token must NOT see f/private/bar, got: {scoped:?}"
);
// Broad `scripts:read` token: still sees every RLS-visible script.
let broad = list_search_paths(port, "BROAD_TOKEN").await;
assert!(broad.contains(&"f/allowed/foo".to_string()));
assert!(
broad.contains(&"f/private/bar".to_string()),
"broad scripts:read token should see all scripts, got: {broad:?}"
);
// Tag-filter-only token is not scope-restricted: unchanged, sees all.
let tag_only = list_search_paths(port, "TAG_TOKEN").await;
assert!(tag_only.contains(&"f/allowed/foo".to_string()));
assert!(tag_only.contains(&"f/private/bar".to_string()));
// Unscoped token (no scopes column set): unchanged, sees all.
let unscoped = list_search_paths(port, "SECRET_TOKEN").await;
assert!(unscoped.contains(&"f/allowed/foo".to_string()));
assert!(unscoped.contains(&"f/private/bar".to_string()));
Ok(())
}
backend/windmill-api-integration-tests/tests/slack_approvals_unauthed.rs
+230
View File
@@ -0,0 +1,230 @@
//! Regression tests for GHSA-vm75-gmpw-rvp9: the unauthenticated `/api/slack` callback must
//! not be drivable into decrypting arbitrary workspace variables.
//!
//! The OpenModal branch reaches `get_slack_token` (a privileged, RLS-bypassing variable
//! decryption). It is now gated by a per-workspace HMAC over (w_id, job_id, path) — the same
//! workspace key used to sign resume URLs. Without a valid signature the request is rejected
//! with 401 before any decryption, even when `SLACK_SIGNING_SECRET` is unset (the default).
use hmac::{Hmac, Mac};
use serde_json::json;
use sha2::Sha256;
use sqlx::{Pool, Postgres};
use uuid::Uuid;
use windmill_test_utils::*;
fn client() -> reqwest::Client {
reqwest::Client::new()
}
/// Re-implementation of the server's `sign_slack_payload` for the positive-control test.
/// The fixture sets `workspace_key.key = 'test-key'` for `test-workspace`.
fn sign(w_id: &str, parts: &[&[u8]]) -> String {
let mut mac = Hmac::<Sha256>::new_from_slice(b"test-key").unwrap();
mac.update(b"slack_payload_v1\0"); // SLACK_PAYLOAD_HMAC_DOMAIN
mac.update(w_id.as_bytes());
for p in parts {
mac.update(b"\0");
mac.update(p);
}
hex::encode(mac.finalize().into_bytes())
}
/// POST an `open_modal` block action to the unauthenticated `/api/slack` callback.
async fn post_open_modal(port: u16, value: serde_json::Value) -> reqwest::Response {
let payload = json!({
"type": "block_actions",
"trigger_id": "trigger-123",
"container": { "message_ts": "0", "channel_id": "C1" },
"actions": [ { "action_id": "open_modal", "value": value.to_string() } ],
});
client()
.post(format!("http://localhost:{port}/api/slack"))
.form(&[("payload", payload.to_string())])
.send()
.await
.unwrap()
}
/// POST a `view_submission` to the unauthenticated `/api/slack` callback with the given
/// private_metadata.
async fn post_view_submission(port: u16, private_metadata: serde_json::Value) -> reqwest::Response {
let payload = json!({
"type": "view_submission",
"view": {
"state": { "values": {} },
"private_metadata": private_metadata.to_string(),
},
});
client()
.post(format!("http://localhost:{port}/api/slack"))
.form(&[("payload", payload.to_string())])
.send()
.await
.unwrap()
}
/// A submission with an unsigned (or tampered) `private_metadata` must be rejected with 401
/// BEFORE the resume/cancel action runs — the signature gate is checked first. The resume_url
/// here is well-formed (so it parses) but never acted upon.
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_view_submission_without_signature_is_rejected(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let job_id = Uuid::new_v4();
let resume_url = format!("/api/w/test-workspace/jobs_u/resume/{job_id}/1/deadbeef");
let resp = post_view_submission(
port,
json!({
"resume_url": resume_url,
"resource_path": "u/admin/secret",
"container": { "message_ts": "0", "channel_id": "C1" },
"hide_cancel": false,
}),
)
.await;
assert_eq!(
resp.status(),
401,
"unsigned submission must be rejected before the resume action"
);
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_open_modal_without_signature_is_rejected(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let job_id = Uuid::new_v4();
// No signature → must be rejected with 401 before any variable lookup. Before the fix
// this reached `get_slack_token` and forced decryption of `u/admin/secret`.
let resp = post_open_modal(
port,
json!({
"w_id": "test-workspace",
"job_id": job_id.to_string(),
"path": "u/admin/secret",
"flow_step_id": "a",
}),
)
.await;
assert_eq!(
resp.status(),
401,
"unsigned OpenModal callback must be rejected"
);
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_open_modal_with_wrong_signature_is_rejected(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let job_id = Uuid::new_v4();
let resp = post_open_modal(
port,
json!({
"w_id": "test-workspace",
"job_id": job_id.to_string(),
"path": "u/admin/secret",
"flow_step_id": "a",
"signature": "deadbeef",
}),
)
.await;
assert_eq!(
resp.status(),
401,
"OpenModal callback with an invalid signature must be rejected"
);
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_open_modal_with_tampered_path_is_rejected(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let job_id = Uuid::new_v4();
// A signature legitimately minted for one path cannot be reused to decrypt another: the
// path is bound into the HMAC.
let signature = sign(
"test-workspace",
&[job_id.to_string().as_bytes(), b"u/admin/legit_resource"],
);
let resp = post_open_modal(
port,
json!({
"w_id": "test-workspace",
"job_id": job_id.to_string(),
"path": "u/admin/some_other_secret",
"flow_step_id": "a",
"signature": signature,
}),
)
.await;
assert_eq!(
resp.status(),
401,
"a signature bound to a different path must not authorize decryption"
);
Ok(())
}
#[sqlx::test(migrations = "../migrations", fixtures("base"))]
async fn test_open_modal_with_valid_signature_passes_the_gate(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let job_id = Uuid::new_v4();
let path = "u/admin/nonexistent_resource";
// A correctly signed payload passes the authorization gate and proceeds to resolve the
// slack resource. The resource does not exist, so the handler returns a generic 400
// ("Invalid Slack callback request") rather than 401 — proving the gate accepted the
// signature (so the fix does not simply reject everything) without echoing the path.
let signature = sign(
"test-workspace",
&[job_id.to_string().as_bytes(), path.as_bytes()],
);
let resp = post_open_modal(
port,
json!({
"w_id": "test-workspace",
"job_id": job_id.to_string(),
"path": path,
"flow_step_id": "a",
"signature": signature,
}),
)
.await;
let status = resp.status();
let body = resp.text().await?;
assert_eq!(
status, 400,
"validly signed callback should pass the gate and 400 on the missing resource, got {status}: {body}"
);
assert!(
!body.contains("nonexistent_resource"),
"error must not echo the probed path: {body}"
);
Ok(())
}

Some files were not shown because too many files have changed in this diff Show More