starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-14 00:02:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into operator-fork-workspace-ruleset

Browse Source 
# Conflicts:
#	backend/ee-repo-ref.txt
#	frontend/src/lib/utils/editInFork.ts
This commit is contained in:
Guilhem Lemouel
2026-10-13 05:57:57 -07:00
parent e6fd4ef877 3d67d3ac6b
commit b694cd3ef1
897 changed files with 41533 additions and 14773 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/DockerfileBackendTests
+1 -1
View File
@@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
# UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
ENV TZ=Etc/UTC
.github/workflows/ai-evals-scheduled.yml
+2 -2
View File
@@ -16,7 +16,7 @@ on:
default: "global cli"
models:
description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for"
default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro"
default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro"
runs:
description: "Runs per case"
default: "3"
@@ -28,7 +28,7 @@ concurrency:
group: ai-evals-scheduled-${{ github.ref }}
env:
MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }}
MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro' }}
RUNS: ${{ inputs.runs || '3' }}
REASONING: ${{ inputs.reasoning }}
# Routes on the cloud instance are prefixed with their workspace.
.github/workflows/backend-test-windows.yml
+2 -2
View File
@@ -63,7 +63,7 @@ jobs:
- uses: denoland/setup-deno@v2
with:
# Pin to the Deno version shipped in the runtime image (Dockerfile) so CI
# Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI
# tests what production runs, instead of floating on the latest v2.x.
deno-version: 2.2.1
@@ -81,7 +81,7 @@ jobs:
- uses: astral-sh/setup-uv@v7
with:
version: "0.11.24"
version: "0.13.0"
- uses: shivammathur/setup-php@v2
with:
.github/workflows/backend-test.yml
+3 -3
View File
@@ -50,7 +50,7 @@ jobs:
dotnet-version: "9.0.x"
- uses: denoland/setup-deno@v2
with:
# Pin to the Deno version shipped in the runtime image (Dockerfile) so CI
# Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI
# tests what production runs, instead of floating on the latest v2.x.
deno-version: 2.2.1
- uses: actions/setup-go@v7
@@ -64,7 +64,7 @@ jobs:
node-version: "24"
- uses: astral-sh/setup-uv@v7
with:
version: "0.11.24"
version: "0.13.0"
- uses: shivammathur/setup-php@v2
with:
php-version: "8.3"
@@ -295,4 +295,4 @@ jobs:
# never reaches it. Pin the target dir (matching the cache step above) so
# its own tests run off this compile rather than a second bundled build.
(cd windmill-duckdb-ffi-internal && export CARGO_TARGET_DIR="$PWD/target" && ./build_dev.sh && cargo test --release -p windmill_duckdb_ffi_internal)
DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger --all -- --nocapture --test-threads=10
DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger,openidconnect --all -- --nocapture --test-threads=10
.github/workflows/benchmark.yml
+14 -14
View File
@@ -27,7 +27,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
@@ -111,7 +111,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
@@ -125,7 +125,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -136,7 +136,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -147,7 +147,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -188,7 +188,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
@@ -202,7 +202,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -213,7 +213,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -224,7 +224,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -235,7 +235,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -246,7 +246,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -257,7 +257,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -268,7 +268,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
steps:
@@ -309,7 +309,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
.github/workflows/npm_on_release.yml
+7 -4
View File
@@ -18,16 +18,19 @@ jobs:
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
publish_chat_sdk:
runs-on: ubicloud-standard-8
# Published by npm trusted publishing (OIDC), which npm accepts only from
# GitHub-hosted runners and npm >= 11.5.1: keep both, there is no token to fall back on.
runs-on: ubuntu-latest
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v7
with:
node-version: "20.x"
node-version: "24.x"
registry-url: "https://registry.npmjs.org"
- run: cd chat-sdk && npm ci && npm run build && npm publish --access public
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
publish_cli:
runs-on: ubicloud-standard-8
steps:
.release-please-manifest.json
+1 -1
View File
@@ -1,3 +1,3 @@
{
".": "1.826.0"
".": "1.830.0"
}
AGENTS.md
+6 -2
View File
@@ -33,12 +33,16 @@ Open-source platform for internal tools, workflows, API integrations, background
administers behind `external_instance` data tables and Ducklake catalogs: its invariants (one
lifecycle lock, managed-object markers, the setup gate, per-cluster roles, fork copy ownership)
and how to run one locally
- **Data table viewer roles**: `docs/datatable-viewer-roles.md`, on what a connection naming no
role holds: the default role as a floor, where the viewer comes from, why an admin's roles are
not the viewer's, the per-role-set logins and the order their rows and logins are written in
- **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow
- **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are
enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option`
fields and a jsonb merge rather than a serde default
- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace setting that lets
operators compose flows, what the composition check must cover, and why it costs a full seat
- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace settings that let
operators compose flows and full-code apps, what the composition check must cover, which
raw-app endpoint stays closed on purpose, and why either right costs a full seat
- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation
scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and
that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates
CHANGELOG.md
+93
View File
@@ -1,5 +1,98 @@
# Changelog
## [1.830.0](https://github.com/windmill-labs/windmill/compare/v1.829.0...v1.830.0) (2026-10-09)
### Features
* add a jobs:run:dependencies token scope for lock generation ([#11643](https://github.com/windmill-labs/windmill/issues/11643)) ([1dc4ed5](https://github.com/windmill-labs/windmill/commit/1dc4ed5e7fb86c9658f1f877e7b92cd9dab5e283))
* let operators compose full-code apps when the workspace grants it ([#11229](https://github.com/windmill-labs/windmill/issues/11229)) ([02e4697](https://github.com/windmill-labs/windmill/commit/02e4697157112581e9e2a9303180077126db714b))
* operators get AI sessions ([#11443](https://github.com/windmill-labs/windmill/issues/11443)) ([338c4e9](https://github.com/windmill-labs/windmill/commit/338c4e921dc3185d886c8b040f253285adf20efb))
* replace the hub_sync script with an opt-in daily server sync ([#11624](https://github.com/windmill-labs/windmill/issues/11624)) ([2b55a8a](https://github.com/windmill-labs/windmill/commit/2b55a8afbc8a86c6cebe97b1cab02c313b986f22))
* retry transient AI provider failures before output reaches readers ([#11623](https://github.com/windmill-labs/windmill/issues/11623)) ([d3ddef0](https://github.com/windmill-labs/windmill/commit/d3ddef0b7ec4705aa578c4eca24b96e9152485bd))
* stop offering deno for new scripts, ship it only on full images ([#11648](https://github.com/windmill-labs/windmill/issues/11648)) ([cda7883](https://github.com/windmill-labs/windmill/commit/cda78830fde1b96bd3de2674e23c245b50e3c051))
### Bug Fixes
* **cli:** count superadmins as able to preserve ownership on push ([#11653](https://github.com/windmill-labs/windmill/issues/11653)) ([70db567](https://github.com/windmill-labs/windmill/commit/70db567a78b1e1bc9e2dcc9ff2139951d778117e))
* harden app secret validation ([#11655](https://github.com/windmill-labs/windmill/issues/11655)) ([8556a7b](https://github.com/windmill-labs/windmill/commit/8556a7bebe093e0bd61c33d67415fd553f0070ad))
* open the chat + menu without waiting on the skills and mcp fetches ([#11645](https://github.com/windmill-labs/windmill/issues/11645)) ([1fea1aa](https://github.com/windmill-labs/windmill/commit/1fea1aa6609d644e02bac8d16186f9a2dfcb63cd))
* say Running on a chat run card only once its job exists ([#11627](https://github.com/windmill-labs/windmill/issues/11627)) ([d046187](https://github.com/windmill-labs/windmill/commit/d046187e3edd01535240c6080b483afe154ae189))
## [1.829.0](https://github.com/windmill-labs/windmill/compare/v1.828.0...v1.829.0) (2026-10-09)
### Features
* add an api base url instance setting for displayed urls ([#11628](https://github.com/windmill-labs/windmill/issues/11628)) ([6acf185](https://github.com/windmill-labs/windmill/commit/6acf1858540e83b65b5218c50c98af692092d22e))
* add Pi template to the AI sandbox picker ([#11619](https://github.com/windmill-labs/windmill/issues/11619)) ([a85d0b7](https://github.com/windmill-labs/windmill/commit/a85d0b76cc494aaa4a484a273ec534bdc2958102))
* agent nits: multi-select, memory-off notice, agent step label ([#11622](https://github.com/windmill-labs/windmill/issues/11622)) ([c52ba2f](https://github.com/windmill-labs/windmill/commit/c52ba2fd30e19986522420ff02fae224a075b4f6))
* export worker cpu, memory and occupancy metrics ([#11629](https://github.com/windmill-labs/windmill/issues/11629)) ([5d90f96](https://github.com/windmill-labs/windmill/commit/5d90f96863b64e9bc88ed35429f8068a6a788d1e))
* let a job see that its worker is draining ([#11612](https://github.com/windmill-labs/windmill/issues/11612)) ([588d846](https://github.com/windmill-labs/windmill/commit/588d8463dfa1c8936b8753b5f0fd794ecb115203))
* let workspace admins delete other users' drafts ([#11644](https://github.com/windmill-labs/windmill/issues/11644)) ([68376fc](https://github.com/windmill-labs/windmill/commit/68376fc011ef6406a820c69217058316514d3f8d))
* per-script resource rollup and heaviest scripts table ([#11631](https://github.com/windmill-labs/windmill/issues/11631)) ([442357d](https://github.com/windmill-labs/windmill/commit/442357dbe8db0c52995572acd7385dedf4efdc42))
* pipeline editor: assets-only view, Fix actions and draft triggers ([#11440](https://github.com/windmill-labs/windmill/issues/11440)) ([3dae553](https://github.com/windmill-labs/windmill/commit/3dae553576ba23b2c1f56e4deac01305c4645c01))
### Bug Fixes
* advertise the actual signing algorithm in OIDC discovery ([#11637](https://github.com/windmill-labs/windmill/issues/11637)) ([98fc65a](https://github.com/windmill-labs/windmill/commit/98fc65a16bbe2bc9515cdd5443304053c4322274))
* ask a workspace for its session backups before pushing to it ([#11641](https://github.com/windmill-labs/windmill/issues/11641)) ([2ea846f](https://github.com/windmill-labs/windmill/commit/2ea846f563f5a77cbd3eae2eddee9e38a3700d00))
* keep instance settings users sub-tabs from being squashed ([#11633](https://github.com/windmill-labs/windmill/issues/11633)) ([fb22e5c](https://github.com/windmill-labs/windmill/commit/fb22e5ce368eba9146e0d51b8fdabb52ca9f68f0))
* keep raw app deploy panel settings shared with the editor ([#11616](https://github.com/windmill-labs/windmill/issues/11616)) ([0a95ea4](https://github.com/windmill-labs/windmill/commit/0a95ea47facaede2545cc4741d4a70c1c847e11a))
* load schedules once per page load, poll critical alerts every 30s ([#11639](https://github.com/windmill-labs/windmill/issues/11639)) ([1b4e686](https://github.com/windmill-labs/windmill/commit/1b4e686b2f5de3138df0c3b6ed627ea0fe0a2f7d))
* name the open workspace settings tab in the page header breadcrumb ([#11617](https://github.com/windmill-labs/windmill/issues/11617)) ([e233908](https://github.com/windmill-labs/windmill/commit/e233908c1e99a246dc7c94fa86ea82c5cbe9d6b3))
* **python:** add missing import-to-package mappings for common modules ([#11638](https://github.com/windmill-labs/windmill/issues/11638)) ([ab8606f](https://github.com/windmill-labs/windmill/commit/ab8606f37921913ef1410da735abb5742c025225))
* send hub script completions to the workspace handlers ([#11613](https://github.com/windmill-labs/windmill/issues/11613)) ([d685332](https://github.com/windmill-labs/windmill/commit/d685332771dcdfc718d7a2077fe1cdb3b323909b))
* set WM_END_USER_EMAIL on direct script and flow runs ([#11636](https://github.com/windmill-labs/windmill/issues/11636)) ([2bdeceb](https://github.com/windmill-labs/windmill/commit/2bdeceba7fb6815cf3150ecb856afb8eb3872e44))
* store a WAC parent's first start and wall duration ([#11625](https://github.com/windmill-labs/windmill/issues/11625)) ([8f28a58](https://github.com/windmill-labs/windmill/commit/8f28a5860595dbe558e6777a06bb97b6ec6a3e67))
* sync back volume files rewritten to the same size ([#11620](https://github.com/windmill-labs/windmill/issues/11620)) ([2758920](https://github.com/windmill-labs/windmill/commit/2758920f1182ddd12e6473a644a4a966a435522e))
* uniform decrypt error and clear stored credentials on logout ([#11632](https://github.com/windmill-labs/windmill/issues/11632)) ([987c321](https://github.com/windmill-labs/windmill/commit/987c321ebcd305df79fa945e941abc2b2e1d3e95))
## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08)
### Features
* add gpt-6.1-sol to the chat, agent steps and weekly ai evals ([#11584](https://github.com/windmill-labs/windmill/issues/11584)) ([8f7e0c7](https://github.com/windmill-labs/windmill/commit/8f7e0c72848b56fa97f6f0141d930c7d15044864))
* add wmill user list and document post-suspend approvers context ([#11600](https://github.com/windmill-labs/windmill/issues/11600)) ([1c2667c](https://github.com/windmill-labs/windmill/commit/1c2667cd5530925893cf69fd622938b6ffa55d1d))
* ask before minting the frontend sdk token in the raw app editor ([#11604](https://github.com/windmill-labs/windmill/issues/11604)) ([6e948b0](https://github.com/windmill-labs/windmill/commit/6e948b07d13b85e9251c3a200bf149f167649bc4))
* improve data table roles and grants UX ([#11562](https://github.com/windmill-labs/windmill/issues/11562)) ([d7d18f0](https://github.com/windmill-labs/windmill/commit/d7d18f00af4e2df8fcb1362c9439d2250d0ce4b6))
* one page header across routes, and one sidebar for operators and developers ([#11310](https://github.com/windmill-labs/windmill/issues/11310)) ([41162b2](https://github.com/windmill-labs/windmill/commit/41162b270c70ddac6a256bde9acf31a629990e24))
### Bug Fixes
* add retry to runScriptByPathAsync and runScriptByHashAsync ([#11485](https://github.com/windmill-labs/windmill/issues/11485)) ([3f3b1c2](https://github.com/windmill-labs/windmill/commit/3f3b1c2c308ddaba377db45308711e7c660d8775))
* describe exponential retry as multiplier × seconds^attempt ([#11546](https://github.com/windmill-labs/windmill/issues/11546)) ([a2e51cd](https://github.com/windmill-labs/windmill/commit/a2e51cdbaafb7d71b8e8bbd818e40c472098f9ba))
* keep capturing job logs after a non-UTF-8 output line ([#11595](https://github.com/windmill-labs/windmill/issues/11595)) ([0e7d974](https://github.com/windmill-labs/windmill/commit/0e7d974ce9b468353ecb6618dace710704690687))
### Performance Improvements
* load public raw apps without the loading screen ([#11608](https://github.com/windmill-labs/windmill/issues/11608)) ([c09764f](https://github.com/windmill-labs/windmill/commit/c09764fa6516badddbdbb520af32e5d1541cf996))
## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07)
### Features
* audit instance settings writes, stable audit OTel target ([#11593](https://github.com/windmill-labs/windmill/issues/11593)) ([0792332](https://github.com/windmill-labs/windmill/commit/0792332859a88b1359ea1be64250d38dd071a0a9))
* default new raw apps to react 19 and ask for the data setup ([#11586](https://github.com/windmill-labs/windmill/issues/11586)) ([818ddd5](https://github.com/windmill-labs/windmill/commit/818ddd5d31bd81507e274dd18022c1cdbd9be842))
* export queue delay as OTel and Prometheus metrics ([#11588](https://github.com/windmill-labs/windmill/issues/11588)) ([7565349](https://github.com/windmill-labs/windmill/commit/75653496e542550039cbe5c413d8fcb9d395cea8))
* pause a worker group without stopping its workers ([#11596](https://github.com/windmill-labs/windmill/issues/11596)) ([95c4b30](https://github.com/windmill-labs/windmill/commit/95c4b3096cf10c1fe59de7a1a6876dfb5f02c39f))
* run AI decision steps on OpenAI's Decisions API ([#11590](https://github.com/windmill-labs/windmill/issues/11590)) ([046df9e](https://github.com/windmill-labs/windmill/commit/046df9e44db2f2fae3059ccd7a3d6c6955ee8971))
### Bug Fixes
* avoid URLSearchParams.keys() so the frontend check passes ([#11599](https://github.com/windmill-labs/windmill/issues/11599)) ([fc6c84c](https://github.com/windmill-labs/windmill/commit/fc6c84c6974bba1850e90a088fad5460c8c93ba3))
* let recover: true from the error handler turn nested failures green ([#11559](https://github.com/windmill-labs/windmill/issues/11559)) ([03e484e](https://github.com/windmill-labs/windmill/commit/03e484e59757c0d4d6f74511d9aa5a1e2f94af67))
* run dependency resolution and build steps under nsjail ([#11609](https://github.com/windmill-labs/windmill/issues/11609)) ([a238d01](https://github.com/windmill-labs/windmill/commit/a238d01ef6a8c285aef6886c974904f24339f7a2))
* scope the python wheel cache to workspaces with their own registry ([#11597](https://github.com/windmill-labs/windmill/issues/11597)) ([14381ce](https://github.com/windmill-labs/windmill/commit/14381cefee6fb2b0ece52fbf0001c2ce61bde1a1))
* type snowflake columns in database manager insert and alter forms ([#11578](https://github.com/windmill-labs/windmill/issues/11578)) ([f2fb156](https://github.com/windmill-labs/windmill/commit/f2fb15603b2e0d11bd5d04dd675212a5e4a7b9d7))
## [1.826.0](https://github.com/windmill-labs/windmill/compare/v1.825.0...v1.826.0) (2026-10-07)
Dockerfile
+1 -4
View File
@@ -83,7 +83,6 @@ COPY /python-client/docs/ /frontend/static/pydocs/
RUN npm run generate-backend-client
ENV NODE_OPTIONS "--max-old-space-size=8192"
ARG VITE_BASE_URL ""
# Must be declared for the build-arg to reach the bundle. See frontend/svelte.config.js.
ARG WM_BUILD_VERSION=""
# Read more about macro in docker/dev.nu
@@ -242,7 +241,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
# Preinstall python runtimes to temp build location (will copy with world-writable perms later)
# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run
@@ -287,8 +286,6 @@ COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so ${APP}/libwindmill_duckdb_ffi_internal.so
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun
# Install windmill CLI
README.md
+2 -2
View File
@@ -150,7 +150,7 @@ You can run scripts locally by passing the right environment variables for the `
- **Frontend**: Svelte 5
- **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation
- **Runtimes**:
- TypeScript/JavaScript: Bun (default) and Deno
- TypeScript/JavaScript: Bun (default); Deno only on the `windmill-full` images
- Python: python3 with uv for dependency management
- Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible
@@ -318,7 +318,7 @@ running options.
This will also avoid compile time issue with sqlx's `query!` macro.
2. (optional, linux only) Install [nsjail](https://github.com/google/nsjail) and have it accessible in
your PATH
3. Install bun, deno and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun`,`/usr/bin/deno`, and
3. Install bun and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun` and
`/usr/local/bin/python3` or set the corresponding environment variables.
4. (optional) Install the [lld linker](https://lld.llvm.org/)
5. Go to `frontend/`:
ai_evals/README.md
+4 -1
View File
@@ -196,7 +196,10 @@ not have read is itself the failure — e.g. `write_variable.value` on a secret
Global (and flow) initial fixtures can seed `workspace.datatables` so the
`list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools
return seeded data during evals. Each entry is
`{ datatable_name, schemas: { <schema>: { <table>: { columns, rows? } } } }`.
`{ datatable_name, schemas: { <schema>: { <table>: { columns, rows? } } } }`,
plus `migrations_enabled: true` and `migrations: [{ timestamp, name, code_up, code_down?, applied? }]`
for a datatable that keeps its schema as migrations, which the migration tools then
read and write in memory.
SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real
database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/
`DELETE` mutate the seeded datatable in place, so a later `list_datatables`,
ai_evals/adapters/frontend/datatableSqlEngine.ts
+11
View File
@@ -30,6 +30,17 @@ export interface BenchmarkDatatableSeed {
[table: string]: BenchmarkDatatableTableSeed
}
}
/** Whether the datatable keeps its schema as migrations. */
migrations_enabled?: boolean
migrations?: BenchmarkDatatableMigrationSeed[]
}
export interface BenchmarkDatatableMigrationSeed {
timestamp: number
name: string
code_up: string
code_down?: string
applied?: boolean
}
export interface DatatableSqlResult {
ai_evals/adapters/frontend/mockBackend.ts
+129 -1
View File
@@ -692,10 +692,138 @@ export function listBenchmarkDatatables(workspace: string): DataTableTables[] |
datatable_name: datatable.datatable_name,
schemas: Object.fromEntries(
Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)])
)
),
...(datatable.migrations_enabled && { migrations_enabled: true })
}))
}
// ============= Datatable migrations =============
function benchmarkDatatable(workspace: string, datatableName: string): BenchmarkDatatableSeed {
const datatable = (benchmarkWorkspaceRunnables.get(workspace)?.datatables ?? []).find(
(entry) => entry.datatable_name === datatableName
)
if (!datatable) {
throw new Error(`datatable "${datatableName}" not found`)
}
return datatable
}
/** Run a migration body through the in-memory engine, one statement at a time and without the
* transaction frame, so the tables it creates show up in later reads. */
function applyBenchmarkMigrationSql(datatable: BenchmarkDatatableSeed, sql: string): void {
// Split on the semicolons outside single-quoted literals.
for (const statement of sql.match(/(?:'(?:[^']|'')*'|[^;'])+/g) ?? []) {
const trimmed = statement.replace(/^\s*--.*$/gm, '').trim()
if (trimmed !== '' && !/^(begin|end|commit)$/i.test(trimmed)) {
applyDatatableSql(datatable, trimmed)
}
}
}
export function getBenchmarkDatatableMigrationsStatus(input: {
workspace: string
datatableName: string
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
if (!datatable.migrations_enabled) {
return { enabled: false, migrations: [] }
}
return {
enabled: true,
migrations: (datatable.migrations ?? []).map(({ applied, ...migration }) => ({
...migration,
status: applied ? ('ran' as const) : ('not_run' as const)
}))
}
}
export function createBenchmarkDatatableMigration(input: {
workspace: string
datatableName: string
requestBody: { name: string; code_up: string; code_down?: string }
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
if (!datatable.migrations_enabled) {
throw new Error(
`Migrations are not enabled for data table '${input.datatableName}'. Enable them first.`
)
}
const migrations = (datatable.migrations ??= [])
const timestamp = Math.max(20260101000000, ...migrations.map((m) => m.timestamp)) + 1
migrations.push({ timestamp, ...input.requestBody })
return { datatable: input.datatableName, timestamp, ...input.requestBody }
}
export function upsertBenchmarkDatatableMigration(input: {
workspace: string
datatableName: string
requestBody: { timestamp: number; name: string; code_up: string; code_down?: string }
}): string {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
const migrations = (datatable.migrations ??= [])
const existing = migrations.find((m) => m.timestamp === input.requestBody.timestamp)
if (existing?.applied) {
throw new Error(
`Migration ${existing.timestamp} on data table '${input.datatableName}' has already been applied and cannot be modified`
)
}
if (existing) {
// Replaced whole, as the server does: a down left out is stored as none.
delete existing.code_down
Object.assign(existing, input.requestBody)
} else {
migrations.push({ ...input.requestBody })
migrations.sort((a, b) => a.timestamp - b.timestamp)
}
return `Upserted migration ${input.requestBody.timestamp} in ${input.datatableName}`
}
export function runBenchmarkDatatableMigrations(input: {
workspace: string
datatableName: string
upTo?: number
only?: number
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
const applied: { version: number; name: string }[] = []
for (const migration of datatable.migrations ?? []) {
if (
migration.applied ||
(input.only !== undefined && migration.timestamp !== input.only) ||
(input.only === undefined && input.upTo !== undefined && migration.timestamp > input.upTo)
) {
continue
}
applyBenchmarkMigrationSql(datatable, migration.code_up)
migration.applied = true
applied.push({ version: migration.timestamp, name: migration.name })
}
return { applied }
}
export function rollbackBenchmarkDatatableMigration(input: {
workspace: string
datatableName: string
only?: number
}) {
const datatable = benchmarkDatatable(input.workspace, input.datatableName)
const last = (datatable.migrations ?? [])
.filter((m) => m.applied && (input.only === undefined || m.timestamp === input.only))
.at(-1)
if (!last) {
return { rolled_back: [] }
}
if (last.code_down === undefined) {
throw new Error(
`Cannot roll back migration ${last.timestamp} (${last.name}): it has no down migration`
)
}
applyBenchmarkMigrationSql(datatable, last.code_down)
last.applied = false
return { rolled_back: [{ version: last.timestamp, name: last.name }] }
}
// ============= DuckLake catalogs and declared metrics =============
/** Seeded DuckLake names, or `null` for a non-benchmark workspace. */
ai_evals/adapters/frontend/vitestAdapter.test.ts
+26 -1
View File
@@ -92,6 +92,11 @@ vi.mock('$lib/gen', async () => {
getBenchmarkCompletedJob,
getBenchmarkCompletedJobResultMaybe,
getBenchmarkDatatableSchema,
getBenchmarkDatatableMigrationsStatus,
createBenchmarkDatatableMigration,
upsertBenchmarkDatatableMigration,
runBenchmarkDatatableMigrations,
rollbackBenchmarkDatatableMigration,
getBenchmarkDraftForUser,
getBenchmarkFlowByPath,
getBenchmarkFlowAllResults,
@@ -400,7 +405,27 @@ vi.mock('$lib/gen', async () => {
schemaName: data.schemaName,
tableName: data.tableName
})
: actual.WorkspaceService.getDataTableTableSchema(data)
: actual.WorkspaceService.getDataTableTableSchema(data),
getDatatableMigrationsStatus: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? getBenchmarkDatatableMigrationsStatus(data)
: actual.WorkspaceService.getDatatableMigrationsStatus(data),
createDatatableMigration: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? createBenchmarkDatatableMigration(data)
: actual.WorkspaceService.createDatatableMigration(data),
upsertDatatableMigration: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? upsertBenchmarkDatatableMigration(data)
: actual.WorkspaceService.upsertDatatableMigration(data),
runDatatableMigrations: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? runBenchmarkDatatableMigrations(data)
: actual.WorkspaceService.runDatatableMigrations(data),
rollbackDatatableMigrations: async (data: any) =>
hasBenchmarkWorkspace(data.workspace)
? rollbackBenchmarkDatatableMigration(data)
: actual.WorkspaceService.rollbackDatatableMigrations(data)
}),
DataMetricService: wrapService(actual.DataMetricService, {
listDataMetrics: async (data: { workspace: string }) =>
ai_evals/cases/global.yaml
+51
View File
@@ -901,6 +901,57 @@
- treats creating the table as a normal SQL statement and does not claim a separate registration step is needed
- does not write a script to create the table
# The same request on a data table that keeps its schema as migrations: the table is
# created by a migration, since plain SQL would leave it out of the history.
- id: global-datatable-schema-change-uses-migration
prompt: |-
Add a new table called refunds to our workspace data table, with an order id and a refund amount.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json
runtime:
maxTurns: 8
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- create_datatable_migration
forbiddenToolsUsed:
- write_script
- deploy_workspace_item
- delete_workspace_item
toolCallArgs:
- tool: create_datatable_migration
field: code_up
stringIncludesAnyOf:
- create table
skipJudge: true
judgeChecklist:
- notices that the data table keeps its schema as migrations
- creates the refunds table through a migration rather than a plain CREATE TABLE statement
- includes an order id and a refund amount column
- gives the migration a down that drops the table
# Row data is not schema: it is written with plain SQL even where migrations are enabled.
- id: global-datatable-row-change-skips-migration
prompt: |-
Mark order number 2 as shipped in our workspace data table.
initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json
runtime:
maxTurns: 12
validate:
draftCountExactly: 0
toolExpect:
requiredToolsUsed:
- exec_datatable_sql
forbiddenToolsUsed:
- create_datatable_migration
- write_script
- deploy_workspace_item
- delete_workspace_item
skipJudge: true
judgeChecklist:
- runs an UPDATE on the orders table setting the status of order id 2 to shipped
- does not create a migration for a change to row data
- id: global-test25-datatable-mutate-rows
prompt: |-
Mark order number 2 as shipped in our workspace data table.
ai_evals/core/models.ts
+9
View File
@@ -148,6 +148,15 @@ export const EVAL_MODELS: EvalModelSpec[] = [
model: "gpt-6-astra",
},
},
{
id: "gpt-6.1-sol",
label: "GPT-6.1 Sol",
aliases: ["gpt-6.1-sol"],
frontend: {
provider: "openai",
model: "gpt-6.1-sol",
},
},
{
id: "gpt-6-sol",
label: "GPT-6 Sol",
ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json
+36
View File
@@ -0,0 +1,36 @@
{
"workspace": {
"datatables": [
{
"datatable_name": "main",
"migrations_enabled": true,
"migrations": [
{
"timestamp": 20260501090000,
"name": "create_orders",
"code_up": "BEGIN;\n\nCREATE TABLE orders (id serial PRIMARY KEY, customer_id int, total numeric, status text, created_at timestamptz DEFAULT now());\n\nEND;",
"code_down": "BEGIN;\n\nDROP TABLE orders;\n\nEND;",
"applied": true
}
],
"schemas": {
"public": {
"orders": {
"columns": {
"id": "int4",
"customer_id": "int4",
"total": "numeric",
"status": "text",
"created_at": "timestamptz"
},
"rows": [
{ "id": 1, "customer_id": 1, "total": 42.5, "status": "shipped", "created_at": "2026-05-01T10:00:00Z" },
{ "id": 2, "customer_id": 2, "total": 19.99, "status": "pending", "created_at": "2026-05-02T11:30:00Z" }
]
}
}
}
}
]
}
}
backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json → backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true)\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>', true)\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde"
"hash": "009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6"
}
backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json → backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730"
"hash": "1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e"
}
backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217"
}
backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM runnable_job_stats s WHERE s.hour >= $1 AND NOT EXISTS (SELECT 1 FROM workspace w WHERE w.id = s.workspace_id)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8"
}
backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b"
}
backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json
Generated
+40
View File
@@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",\n COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\",\n COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",\n COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"\n FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "flows!",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "apps!",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "schedules!",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "triggers!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
null,
null,
null
]
},
"hash": "1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80"
}
backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb",
"Text",
"Varchar",
"Bool",
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead"
}
backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO runnable_job_stats\n (hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n SELECT s.hour, s.workspace_id, s.runnable_path, s.worker_group, s.job_count,\n s.total_duration_ms, s.max_memory_peak, s.sum_memory_peak,\n s.memory_sample_count, s.total_cpu_ms\n FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::int[], $6::bigint[],\n $7::int[], $8::bigint[], $9::int[], $10::bigint[])\n AS s(hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n JOIN workspace w ON w.id = s.workspace_id\n ORDER BY s.hour, s.workspace_id, s.runnable_path, s.worker_group\n ON CONFLICT (hour, workspace_id, runnable_path, worker_group) DO UPDATE SET\n job_count = runnable_job_stats.job_count + EXCLUDED.job_count,\n total_duration_ms = runnable_job_stats.total_duration_ms + EXCLUDED.total_duration_ms,\n max_memory_peak = GREATEST(runnable_job_stats.max_memory_peak, EXCLUDED.max_memory_peak),\n sum_memory_peak = runnable_job_stats.sum_memory_peak + EXCLUDED.sum_memory_peak,\n memory_sample_count = runnable_job_stats.memory_sample_count + EXCLUDED.memory_sample_count,\n total_cpu_ms = runnable_job_stats.total_cpu_ms + EXCLUDED.total_cpu_ms\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8Array",
"TextArray",
"TextArray",
"TextArray",
"Int4Array",
"Int8Array",
"Int4Array",
"Int8Array",
"Int4Array",
"Int8Array"
]
},
"nullable": []
},
"hash": "24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875"
}
backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json → backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -8,5 +8,5 @@
},
"nullable": []
},
"hash": "7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877"
"hash": "344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d"
}
backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Jsonb",
"Text",
"Text",
"Bool",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34"
}
backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json → backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95"
"hash": "42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779"
}
backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT canceled_by IS NOT NULL FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": [
null
]
},
"hash": "58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0"
}
backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json → backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4"
"hash": "69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927"
}
backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json
Generated
+45
View File
@@ -0,0 +1,45 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e"
}
backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json
Generated
-38
View File
@@ -1,38 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n id,\n runnable_path,\n permissioned_as_email,\n parent_job\n FROM v2_job\n WHERE workspace_id = 'test-workspace'\n AND permissioned_as_email = 'success_handler@windmill.dev'\n ORDER BY created_at DESC\n LIMIT 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "permissioned_as_email",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "parent_job",
"type_info": "Uuid"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
true,
false,
true
]
},
"hash": "71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496"
}
backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "SELECT wp.draining FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN worker_ping wp ON wp.worker = q.worker\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n AND ($3::text[] IS NULL OR j.tag = ANY($3))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Uuid",
"Text",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13"
}
backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json → backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e"
"hash": "8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0"
}
backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ($1, 'true'::jsonb) ON CONFLICT (name) DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar"
]
},
"nullable": []
},
"hash": "8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97"
}
backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json → backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2",
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2",
"describe": {
"columns": [
{
@@ -31,5 +31,5 @@
null
]
},
"hash": "7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec"
"hash": "8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3"
}
backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "coalesce",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683"
}
backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json
Generated
-31
View File
@@ -1,31 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d"
}
backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json → backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json
Generated
+2 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS (SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running)",
"query": "SELECT EXISTS (SELECT 1 FROM global_settings WHERE name = $1)",
"describe": {
"columns": [
{
@@ -11,7 +11,6 @@
],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
@@ -19,5 +18,5 @@
null
]
},
"hash": "ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e"
"hash": "966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a"
}
backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json
Generated
-33
View File
@@ -1,33 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5"
}
backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM runnable_job_stats WHERE hour < $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e"
}
backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d"
}
backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665"
}
backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "coalesce",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c"
}
backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json → backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c"
"hash": "b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a"
}
backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
null
]
},
"hash": "b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37"
}
backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM token WHERE email = $1 AND label = 'session' AND token_hash != $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877"
}
backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json
Generated
+74
View File
@@ -0,0 +1,74 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT workspace_id, runnable_path, worker_group,\n SUM(job_count)::bigint AS \"job_count!\",\n SUM(total_duration_ms)::bigint AS \"total_duration_ms!\",\n MAX(max_memory_peak) AS \"max_memory_peak!\",\n SUM(sum_memory_peak)::bigint AS \"sum_memory_peak!\",\n SUM(memory_sample_count)::bigint AS \"memory_sample_count!\",\n SUM(total_cpu_ms)::bigint AS \"total_cpu_ms!\"\n FROM runnable_job_stats\n WHERE hour >= $1\n AND ($2::text IS NULL OR worker_group = $2)\n AND ($3::text IS NULL OR workspace_id = $3)\n GROUP BY workspace_id, runnable_path, worker_group\n ORDER BY CASE $4::text\n WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric\n WHEN 'cpu' THEN SUM(total_cpu_ms)\n ELSE SUM(total_duration_ms)\n END DESC, workspace_id, runnable_path, worker_group\n LIMIT $5\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "worker_group",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "job_count!",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "total_duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "max_memory_peak!",
"type_info": "Int4"
},
{
"ordinal": 6,
"name": "sum_memory_peak!",
"type_info": "Int8"
},
{
"ordinal": 7,
"name": "memory_sample_count!",
"type_info": "Int8"
},
{
"ordinal": 8,
"name": "total_cpu_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Text",
"Text",
"Text",
"Int8"
]
},
"nullable": [
false,
false,
false,
null,
null,
null,
null,
null,
null
]
},
"hash": "c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477"
}
backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json
Generated
-33
View File
@@ -1,33 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "jobs_executed",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool"
]
},
"nullable": [
false
]
},
"hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc"
}
backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json → backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -8,5 +8,5 @@
},
"nullable": []
},
"hash": "cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20"
"hash": "cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74"
}
backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json
Generated
+43
View File
@@ -0,0 +1,43 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8"
}
backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM runnable_job_stats WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8"
}
backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT q.id FROM v2_job_queue q\n JOIN native_retry_attempt n ON n.job_id = q.id\n JOIN v2_job r ON r.id = q.id\n JOIN v2_job a ON a.id = r.parent_job\n WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Uuid"
}
],
"parameters": {
"Left": [
"UuidArray",
"Text"
]
},
"nullable": [
false
]
},
"hash": "de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429"
}
backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd"
}
backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET draining = true WHERE worker = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919"
}
backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json
Generated
+34
View File
@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n RETURNING jobs_executed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "jobs_executed",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Text",
"TextArray",
"Varchar",
"Varchar",
"TextArray",
"Varchar",
"Int8",
"Int8",
"Text",
"Bool",
"Bool"
]
},
"nullable": [
false
]
},
"hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b"
}
backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323"
}
backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json → backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1 AND j.parent_job IS NULL\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING",
"query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING",
"describe": {
"columns": [],
"parameters": {
@@ -12,5 +12,5 @@
},
"nullable": []
},
"hash": "e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258"
"hash": "eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9"
}
backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json → backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT COALESCE((ws.operator_settings->>'builder_flows')::boolean, false)) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT (COALESCE((ws.operator_settings->>'builder_flows')::boolean, false) OR COALESCE((ws.operator_settings->>'builder_apps')::boolean, false))) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
"describe": {
"columns": [
{
@@ -34,5 +34,5 @@
null
]
},
"hash": "cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b"
"hash": "efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b"
}
backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed"
}
backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json
Generated
+59
View File
@@ -0,0 +1,59 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3\n AND ($4::text IS NULL OR email = $4)\n RETURNING 1 as \"one!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "one!",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "draft_kind",
"kind": {
"Enum": [
"script",
"flow",
"app",
"raw_app",
"resource",
"variable",
"trigger_schedule",
"trigger_webhook",
"trigger_default_email",
"trigger_email",
"trigger_http",
"trigger_websocket",
"trigger_postgres",
"trigger_kafka",
"trigger_nats",
"trigger_mqtt",
"trigger_sqs",
"trigger_gcp",
"trigger_azure",
"trigger_poll",
"trigger_cli",
"trigger_nextcloud",
"trigger_google",
"trigger_github",
"data_pipeline",
"trigger_amqp"
]
}
}
},
"Text"
]
},
"nullable": [
null
]
},
"hash": "f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18"
}
backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now() WHERE global_settings.value IS DISTINCT FROM EXCLUDED.value",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb"
]
},
"nullable": []
},
"hash": "f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa"
}
backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json → backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -107,6 +107,11 @@
"ordinal": 20,
"name": "native_mode",
"type_info": "Bool"
},
{
"ordinal": 21,
"name": "draining",
"type_info": "Bool"
}
],
"parameters": {
@@ -138,8 +143,9 @@
true,
true,
true,
false,
false
]
},
"hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca"
"hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c"
}
backend/AGENTS.md
+1 -1
View File
@@ -214,7 +214,7 @@ Whole classes of defect compile and unit-test clean:
- **Stack overflow from a large buffer in an async block.** An array declared across an
`.await` is baked into the future's state; once that future is boxed a few layers deep
by the job poller, two 16 KB arrays abort the worker *process* (`thread
'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers
'tokio-rt-worker' has overflowed its stack`). Heap-allocate read buffers
(`vec![0u8; N]`, not `[0u8; N]`).
- Deadlocks from draining only one of a child's pipes, missed cancellation or timeout
propagation, and anything depending on the real engine's output format.
backend/Cargo.lock
Generated
+1533 -846
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+35 -34
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.826.0"
version = "1.830.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.826.0"
version = "1.830.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -156,7 +156,7 @@ bedrock = ["windmill-ai/bedrock", "windmill-api/bedrock", "windmill-worker/bedro
mqtt_trigger = ["windmill-api/mqtt_trigger"]
amqp_trigger = ["windmill-api/amqp_trigger"]
native_trigger = ["windmill-api/native_trigger"]
sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"]
sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "openidconnect"]
gcp_trigger = ["windmill-api/gcp_trigger"]
azure_trigger = ["windmill-api/azure_trigger"]
smtp = ["windmill-api/smtp", "windmill-common/smtp", "windmill-queue/smtp"]
@@ -281,6 +281,7 @@ windmill-api = { workspace = true, default-features = false }
windmill-api-agent-workers = { workspace = true, optional = true }
windmill-api-scripts.workspace = true
windmill-api-settings.workspace = true
windmill-audit.workspace = true
windmill-worker.workspace = true
windmill-indexer = { workspace = true, optional = true }
windmill-autoscaling = { workspace = true, optional = true }
@@ -451,7 +452,7 @@ headers = "^0"
hyper = { version = "^1", features = ["full"] }
hyper-tls = "^0.6"
hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] }
tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
tokio = { version = "=1.53.2", features = ["full", "tracing", "time"] }
tokio-stream = { version = "0.1.17" }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] }
@@ -488,9 +489,11 @@ arc-swap = "1"
ed25519-dalek = { version = "2", features = ["rand_core"] }
magic-crypt = "^3"
git-version = "^0"
malachite = "=0.4.18"
malachite-bigint = "=0.2.0"
rustpython-parser = "^0"
# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly,
# and to the same release.
ruff_python_parser = "=0.0.17"
ruff_python_ast = "=0.0.17"
ruff_text_size = "=0.0.17"
pep440_rs = "0.7.3"
php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" }
cron = "^0"
@@ -519,33 +522,29 @@ aws-sdk-rds = "^1"
async-trait = "0.1.88"
v8 = "=137.1.0" # Exact version NOTE: Do not forget to update version and hash in flake.nix
# deno_* pin set: deno v2.4.0 base, with deno_ast force-overridden to =0.51.0.
# Rationale: deno_ast 0.51.0 is the first version pulling swc_common =14.0.4,
# the first swc_common patch that dropped `pub use serde::__private as serde;`
# (the line that capped our workspace serde pin at =1.0.220). v2.4.0's other
# pins keep deno_tls at 0.196.0 which uses permissive `rustls ^0.23.11`,
# compatible with aws-sdk-bedrockruntime's `^0.23.31` requirement. deno_tls
# 0.198+ tightened that to exact `=0.23.28`, which would have made any
# meaningful deno bump resolver-impossible against aws-sdk.
deno_fetch = "0.233.0"
deno_tls = "0.196.0"
deno_console = "0.209.0"
deno_url = "0.209.0"
deno_webidl = "0.209.0"
deno_web = "0.240.0"
# Sibling release of the pinned deno_core 0.352 / deno_web 0.240 stack
# (its deps are deno_core ^0.352, deno_web ^0.240, deno_error =0.6.1). A newer
# deno_crypto would force bumping the whole deno stack.
deno_crypto = "0.223.0"
deno_io = "0.119.0"
deno_fs = "0.119.0"
deno_net = "0.201.0"
deno_core = "0.352.0"
# deno_* pin set: the crates deno published together on 2026-09-16 (deno_core
# 0.412).
# V8 is pinned exactly because flake.nix fetches the prebuilt archive of that
# version: update its version and hash there with any bump. No default features,
# so the pin leaves the feature set deno_core picks untouched.
v8 = { version = "=150.4.0", default-features = false }
# These release as a matched set: each pins its siblings, deno_error exactly,
# and deno_tls pins rustls exactly (=0.23.40, the workspace `rustls` pin below
# must equal it). A bump moves all of them at once.
# deno_ast is independent of that set and stays at 0.51.0: a newer one pulls
# swc_common 17, which would force bumping every swc_* pin the parsers use.
deno_fetch = "0.283.0"
deno_tls = "0.246.0"
deno_webidl = "0.259.0"
deno_web = "0.290.0"
deno_crypto = "0.273.0"
deno_net = "0.251.0"
deno_core = "0.412.0"
deno_ast = { version = "=0.51.0", features = ["transpiling"] }
deno_permissions = "0.68.0"
deno_telemetry = "0.31.0"
deno_error = "=0.6.1"
deno_permissions = "0.118.0"
deno_telemetry = "0.81.0"
deno_error = "=0.7.1"
sys_traits = { version = "0.1.28", features = ["real", "libc", "winapi"] }
rustls-pemfile = "2.2.0"
# only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed
@@ -596,9 +595,11 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] }
const-str = "0.5"
constant_time_eq = "0.3.1"
rsa = "^0"
p256 = "0.13"
spki = { version = "0.7", features = ["pem"] }
pkcs1 = "0.7"
aes-gcm = "0.10.3"
hkdf = "0.12.4"
async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] }
once_cell = "1.17.1"
dashmap = "6.1.0"
@@ -682,7 +683,7 @@ aws-sdk-sqs = "^1.89.0"
aws-sdk-sts = "^1.91.0"
aws-sdk-sso = "=1.77.0"
aws-sdk-ssooidc = "=1.78.0"
rustls = "=0.23.35"
rustls = "=0.23.40"
async-once-cell = "0.5.4"
size = "0.5.0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
44e6aa910f4a6db94532b2d08919721d86d22bc8
ac721c18c37bb0fd41aae17e8b4439c09168511f
backend/migrations/20261008070335_worker_ping_draining.down.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining;
backend/migrations/20261008070335_worker_ping_draining.up.sql
+1
View File
@@ -0,0 +1 @@
ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE;
backend/migrations/20261008165042_runnable_job_stats.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS runnable_job_stats;
backend/migrations/20261008165042_runnable_job_stats.up.sql
+17
View File
@@ -0,0 +1,17 @@
-- Hourly resource rollup per runnable and worker group, fed by the workers.
-- Memory figures are in kB, like v2_job_completed.memory_peak.
-- No foreign key to workspace: every worker upserts here, and the key would cost each row
-- a lookup and a lock on its workspace. Workspace deletion removes the rows itself.
CREATE TABLE IF NOT EXISTS runnable_job_stats (
hour BIGINT NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
runnable_path VARCHAR(255) NOT NULL,
worker_group TEXT NOT NULL,
job_count INTEGER NOT NULL DEFAULT 0,
total_duration_ms BIGINT NOT NULL DEFAULT 0,
max_memory_peak INTEGER NOT NULL DEFAULT 0,
sum_memory_peak BIGINT NOT NULL DEFAULT 0,
memory_sample_count INTEGER NOT NULL DEFAULT 0,
total_cpu_ms BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (hour, workspace_id, runnable_path, worker_group)
);
backend/migrations/20261012180101_datatable_role_set.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS datatable_role_set;
backend/migrations/20261012180101_datatable_role_set.up.sql
+21
View File
@@ -0,0 +1,21 @@
-- The logins behind a data table connection made as several roles at once: one Postgres login per
-- distinct set of data table roles, a member of each. Which roles a login holds is its
-- memberships on the cluster, and its name is derived from their catalog ids, so a row only has to
-- carry the credential.
--
-- A row is written after its login exists and removed before the login is dropped, so a row always
-- names a login that can authenticate with `pwd`.
CREATE TABLE datatable_role_set (
cluster VARCHAR(20) NOT NULL CHECK (cluster IN ('instance', 'external_instance')),
name VARCHAR(63) NOT NULL,
-- Generated by Windmill, never entered by anyone, and never leaves the server.
pwd TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
-- Recorded at most once a day. A login unused for long enough is dropped, and made again if
-- its set of roles is asked for later.
last_used_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (cluster, name)
);
GRANT ALL ON datatable_role_set TO windmill_user;
GRANT ALL ON datatable_role_set TO windmill_admin;
backend/migrations/20261012213357_widen_variable_value.down.sql
+12
View File
@@ -0,0 +1,12 @@
DO $$
DECLARE
too_long BIGINT;
BEGIN
SELECT count(*) INTO too_long FROM variable WHERE length(value) > 15000;
IF too_long > 0 THEN
RAISE EXCEPTION 'cannot narrow variable.value to VARCHAR(15000): % row(s) are longer than 15000 characters', too_long;
END IF;
END $$;
ALTER TABLE variable DROP CONSTRAINT IF EXISTS variable_value_length;
ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15000);
backend/migrations/20261012213357_widen_variable_value.up.sql
+8
View File
@@ -0,0 +1,8 @@
-- AES-256-GCM ciphertext is 32 bytes (magic, nonce, tag) longer than legacy CBC before base64,
-- so the longest CBC value that fits 15000 chars is exactly 15040 chars once re-encrypted.
ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15040);
-- Only GCM secrets (base64 of the `wmA2` magic starts `d21BM`) may use the extra room: a longer
-- CBC value could never be re-encrypted to fit, and would fail every rotation of its workspace key.
ALTER TABLE variable ADD CONSTRAINT variable_value_length
CHECK (length(value) <= 15000 OR (is_secret AND value LIKE 'd21BM%'));
backend/parsers/windmill-parser-py-asset/Cargo.toml
+2 -2
View File
@@ -11,6 +11,6 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql-asset.workspace = true
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
anyhow.workspace = true
backend/parsers/windmill-parser-py-asset/src/lib.rs
+168 -96
View File
@@ -1,5 +1,8 @@
use rustpython_ast::{Constant, Expr, ExprConstant, Visitor};
use rustpython_parser::{ast::Suite, Parse};
use ruff_python_ast::{
visitor::{walk_expr, walk_stmt, Visitor},
Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign,
};
use ruff_python_parser::parse_module;
use std::collections::HashMap;
use windmill_parser::asset_parser::{
asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind,
@@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{
use AssetUsageAccessType::*;
pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
let ast = Suite::parse(input, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(input)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() };
ast.into_iter()
.for_each(|stmt| assets_finder.visit_stmt(stmt));
assets_finder.visit_body(&ast);
for (kind, path, _) in assets_finder.var_identifiers.into_values() {
// if a db = wmill.datatable() was never used (e.g db.query(...)),
@@ -43,13 +46,72 @@ struct AssetsFinder {
var_identifiers: HashMap<String, (AssetKind, VarAssetName, VarAssetSchema)>,
}
impl Visitor for AssetsFinder {
impl<'a> Visitor<'a> for AssetsFinder {
fn visit_stmt(&mut self, stmt: &'a Stmt) {
if let Stmt::Assign(node) = stmt {
self.track_assignment(node);
}
// Continue with generic visit to catch any other assets in the statement
walk_stmt(self, stmt);
}
fn visit_expr(&mut self, expr: &'a Expr) {
match expr {
// A detected asset call does not recurse,
// so this is only reached when no further context was found
Expr::StringLiteral(s) => self.visit_str(s.value.to_str()),
Expr::FString(f) => {
// Literal text between two interpolations reads as one string, across the
// pieces of an implicitly concatenated f-string too.
let mut literal = String::new();
for part in &f.value {
match part {
FStringPartRef::Literal(s) => literal.push_str(&s.value),
FStringPartRef::FString(f) => {
for element in &f.elements {
match element {
InterpolatedStringElement::Literal(l) => {
literal.push_str(&l.value)
}
InterpolatedStringElement::Interpolation(_) => {
self.visit_str(&std::mem::take(&mut literal))
}
}
}
}
}
}
self.visit_str(&literal);
walk_expr(self, expr);
}
Expr::Call(node) => {
if self.visit_expr_call_inner(node).is_err() {
walk_expr(self, expr);
}
}
_ => walk_expr(self, expr),
}
}
}
impl AssetsFinder {
fn visit_str(&mut self, s: &str) {
if let Some((kind, path)) = parse_asset_syntax(s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
// Handle assignment statements like: x = wmill.datatable('name')
fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) {
fn track_assignment(&mut self, node: &StmtAssign) {
// Check if the value is a call to a tracked function
if let Some(Expr::Name(expr_name)) = node.targets.first() {
// Remove any tracked variables with that name in case of reassignment
let Ok(var_name) = expr_name.id.parse::<String>();
let var_name = expr_name.id.to_string();
let removed = self.var_identifiers.remove(&var_name);
// if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)),
// we still want to register the asset as unknown access type
@@ -69,59 +131,12 @@ impl Visitor for AssetsFinder {
if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) {
// Track target variable
let Ok(var_name) = expr_name.id.parse::<String>();
self.var_identifiers
.insert(var_name, (kind.clone(), name.clone(), schema.clone()));
}
}
// Continue with generic visit to catch any other assets in the expression
self.generic_visit_stmt_assign(node);
}
// visit_call_expr will not recurse if it detects an asset,
// so this will only be called when no further context was found
fn visit_expr_constant(&mut self, node: ExprConstant) {
match node.value {
Constant::Str(s) => {
if let Some((kind, path)) = parse_asset_syntax(&s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
_ => self.generic_visit_expr_constant(node),
}
}
fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) {
match self.visit_expr_call_inner(&node) {
Ok(_) => {}
Err(_) => {
// Check keyword arguments for assets before falling back to generic visit
for keyword in &node.keywords {
if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) =
&keyword.value
{
if let Some((kind, path)) = parse_asset_syntax(s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
}
self.generic_visit_expr_call(node);
}
}
}
}
impl AssetsFinder {
/// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc.
fn extract_asset_from_call(
&self,
@@ -147,18 +162,20 @@ impl AssetsFinder {
// Get the first argument (the asset name)
let name = call
.arguments
.args
.first()
.or_else(|| {
// Try keyword argument 'name' or 'path'
call.keywords
call.arguments
.keywords
.iter()
.find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path")))
.map(|kw| &kw.value)
})
.and_then(|first_arg| {
if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg {
Some(name.clone())
if let Expr::StringLiteral(name) = first_arg {
Some(name.value.to_str().to_string())
} else {
None
}
@@ -179,44 +196,39 @@ impl AssetsFinder {
Some((kind, name, schema))
}
fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> {
fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> {
let ident: String = node
.func
.as_name_expr()
.and_then(|o| o.id.parse().ok())
.map(|o| o.id.to_string())
.or_else(|| {
node.func
.as_attribute_expr()
.and_then(|attr| attr.attr.parse().ok())
.map(|attr| attr.attr.to_string())
})
.ok_or(())?;
// `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()`
let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute {
value, ..
}) = node.func.as_ref()
{
if let Expr::Name(expr_name) = value.as_ref() {
expr_name.id.parse().map_err(|_| ())?
} else {
return Err(());
}
} else {
return Err(());
};
let obj_name: String = node
.func
.as_attribute_expr()
.and_then(|attr| attr.value.as_name_expr())
.map(|name| name.id.to_string())
.ok_or(())?;
if obj_name == "wmill" {
// Continue
} else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) {
if ident == "query" {
let expr_name = node.args.get(0).or_else(|| {
node.keywords
let expr_name = node.arguments.args.get(0).or_else(|| {
node.arguments
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some("name"))
.map(|kw| &kw.value)
});
let sql = match expr_name {
Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql,
Some(Expr::StringLiteral(sql)) => sql.value.to_str(),
_ => return Err(()),
};
// We use the SQL parser to detect RW, specific tables, etc.
@@ -224,7 +236,7 @@ impl AssetsFinder {
*kind,
path,
schema.as_deref(),
&sql,
sql,
);
match sql_assets {
Ok(Some(sql_assets)) => self.assets.extend(sql_assets),
@@ -252,9 +264,10 @@ impl AssetsFinder {
};
let arg_val = match arg {
Arg(i, name) => node.args.get(i).or_else(|| {
Arg(i, name) => node.arguments.args.get(i).or_else(|| {
// Get arg by name
node.keywords
node.arguments
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.map(|kw| &kw.value)
@@ -270,8 +283,9 @@ impl AssetsFinder {
self.assets
.push(ParseAssetsResult { kind, path, access_type, columns: None });
}
Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => {
let path = parse_asset_syntax(&value, false)
Some(Expr::StringLiteral(value)) => {
let value = value.value.to_str();
let path = parse_asset_syntax(value, false)
.map(|(_, p)| p)
.unwrap_or(std::borrow::Cow::Borrowed(value));
self.assets.push(ParseAssetsResult {
@@ -291,26 +305,24 @@ impl AssetsFinder {
struct Arg(usize, &'static str);
/// Extract a string-literal keyword argument, e.g. `s3="value"` in a call.
fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option<String> {
fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option<String> {
keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.and_then(|kw| match &kw.value {
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()),
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
}
/// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`.
fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String> {
dict.keys
fn dict_str_value(dict: &ExprDict, name: &str) -> Option<String> {
dict.items
.iter()
.zip(dict.values.iter())
.find_map(|(key, value)| match (key.as_ref()?, value) {
(
Expr::Constant(ExprConstant { value: Constant::Str(k), .. }),
Expr::Constant(ExprConstant { value: Constant::Str(v), .. }),
) if k.as_str() == name => Some(v.clone()),
.find_map(|item| match (item.key.as_ref()?, &item.value) {
(Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => {
Some(v.value.to_str().to_string())
}
_ => None,
})
}
@@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String>
/// matches the TS object form and the `# on s3:///…` trigger form exactly.
fn s3_object_arg_path(expr: &Expr) -> Option<String> {
let uri = match expr {
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
Expr::StringLiteral(s) => {
let s = s.value.to_str();
match s
.strip_prefix("s3://")
.and_then(|rest| rest.split_once('/'))
{
Some((_, key)) if !key.is_empty() => s.clone(),
Some((_, key)) if !key.is_empty() => s.to_string(),
_ => return None,
}
}
@@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option<String> {
if func_name != "S3Object" {
return None;
}
let key = keyword_str_value(&call.keywords, "s3")?;
let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default();
let key = keyword_str_value(&call.arguments.keywords, "s3")?;
let storage =
keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default();
format!("s3://{storage}/{key}")
}
Expr::Dict(dict) => {
@@ -822,4 +836,62 @@ def main():
},])
);
}
#[test]
fn test_py_asset_parser_recent_syntax() {
let input = r#"
import wmill
lazy import polars
type Rows = list[dict]
def main[T](name: str):
try:
with wmill.load_s3_file_reader('s3:///in.csv') as f:
f.read()
except ValueError, TypeError:
pass
greeting = t"hello {name}"
return f"{wmill.get_resource("u/user/pg")["host"]}"
"#;
let s = parse_assets(input).map(|o| o.assets);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/in.csv".to_string(),
access_type: Some(R),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::Resource,
path: "u/user/pg".to_string(),
access_type: None,
columns: None,
},
])
);
}
#[test]
fn test_py_asset_parser_schema_qualified_db_without_table() {
let input = r#"
import wmill
def main():
db = wmill.datatable('dt:sch')
db.query("ATTACH 'ducklake://other' AS o")
db.query("SELECT * FROM t")
"#;
let s = parse_assets(input).map(|o| o.assets);
assert_eq!(
s.map_err(|e| e.to_string())
.map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::<Vec<_>>()),
Ok(vec![
(AssetKind::DataTable, "dt".to_string()),
(AssetKind::DataTable, "dt/sch.t".to_string()),
(AssetKind::Ducklake, "other".to_string()),
])
);
}
}
backend/parsers/windmill-parser-py-imports/Cargo.toml
+2 -1
View File
@@ -23,7 +23,8 @@ tracing.workspace = true
[dependencies]
windmill-parser.workspace = true
rustpython-parser.workspace = true
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
phf.workspace = true
itertools.workspace = true
serde_json.workspace = true
backend/parsers/windmill-parser-py-imports/src/lib.rs
+46 -37
View File
@@ -13,9 +13,9 @@ use async_recursion::async_recursion;
use itertools::Itertools;
use lazy_static::lazy_static;
#[cfg(not(target_arch = "wasm32"))]
use std::str::FromStr;
#[cfg(not(target_arch = "wasm32"))]
use std::collections::HashMap;
#[cfg(not(target_arch = "wasm32"))]
use std::str::FromStr;
use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP};
#[cfg(not(target_arch = "wasm32"))]
@@ -23,11 +23,8 @@ use regex::Regex;
#[cfg(target_arch = "wasm32")]
use regex_lite::Regex;
use rustpython_parser::{
ast::{Stmt, StmtImport, StmtImportFrom, Suite},
text_size::TextRange,
Parse,
};
use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom};
use ruff_python_parser::parse_module;
#[cfg(not(target_arch = "wasm32"))]
use sqlx::{Pool, Postgres};
#[cfg(not(target_arch = "wasm32"))]
@@ -160,6 +157,7 @@ pub struct ImportPin {
}
pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImport>> {
let full_code = code;
// Use regex to safely find the main function definition
let mut code = DEF_MAIN_RE
.split(code)
@@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
// This is needed because we've split off the real main function above
let code_with_fake_main = format!("{}\n\ndef main(): pass", code);
let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| {
anyhow::anyhow!("Error parsing code for imports: {}", e.to_string())
})?;
let (code, ast) = match parse_module(&code_with_fake_main) {
Ok(parsed) => (code, parsed.into_suite()),
// The split lands on any line reading like the definition of `main`, one inside a
// string included: what precedes it is then not a module, while the whole file is.
Err(e) => match parse_module(full_code) {
Ok(parsed) => (full_code.to_string(), parsed.into_suite()),
Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()),
},
};
// Note: We're still using the original code for finding pins,
// as the TextRange values from the parsed AST would be based on code_with_fake_main
// but we want to match against the original code
let find_pin = |range: TextRange, key: String| {
let find_pin = |stmt_end: usize, key: String| {
let hs = code
.chars()
.skip(range.end().to_usize())
.skip(stmt_end)
.take_while(|e| *e != '\n')
.collect::<String>();
@@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
let mut nimports: Vec<NImport> = ast
.into_iter()
.filter_map(|x| match x {
Stmt::Import(StmtImport { names, range }) => names
Stmt::Import(StmtImport { names, range, .. }) => names
.get(0)
.and_then(|al| find_pin(range, al.name.to_string()))
.and_then(|al| find_pin(range.end().to_usize(), al.name.to_string()))
.or(Some(
names
.into_iter()
@@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
.flatten()
.collect::<Vec<NImport>>(),
)),
Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
Some(process_import(
module.map(|x| x.to_string()),
path,
i.to_usize(),
))
}
Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some(
process_import(module.map(|x| x.to_string()), path, level as usize),
),
Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin(
range,
range.end().to_usize(),
module.clone().map(|x| x.to_string()).unwrap_or_default(),
)
.or(Some(process_import(module.map(|x| x.to_string()), path, 0))),
@@ -351,13 +351,16 @@ async fn parse_python_imports_inner(
temp_script_refs: &Option<HashMap<String, String>>,
) -> error::Result<HashMap<String, NImportResolved>> {
tracing::debug!("Parsing python imports for path: {}", path);
let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
let PythonAnnotations { py310, py311, py312, py313, py314, py315, .. } =
PythonAnnotations::parse(&code);
tracing::debug!(
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}",
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}, py314: {}, py315: {}",
py310,
py311,
py312,
py313
py313,
py314,
py315
);
let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> {
@@ -373,6 +376,8 @@ async fn parse_python_imports_inner(
push_version_specifiers(py311, "==3.11.*".to_owned())?;
push_version_specifiers(py312, "==3.12.*".to_owned())?;
push_version_specifiers(py313, "==3.13.*".to_owned())?;
push_version_specifiers(py314, "==3.14.*".to_owned())?;
push_version_specifiers(py315, "==3.15.*".to_owned())?;
for x in code.lines() {
if x.starts_with("# py:") || x.starts_with("#py:") {
@@ -513,8 +518,14 @@ async fn parse_python_imports_inner(
let mut nested = match n {
NImport::Relative(rpath) => {
// First try to get content from temp_script_refs cache if available
let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) {
tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash);
let code_from_cache = if let Some(hash) =
temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath))
{
tracing::debug!(
"Found relative import '{}' in temp_script_refs with hash '{}'",
rpath,
hash
);
match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await {
Ok(content) => Some(content),
Err(e) => {
@@ -529,19 +540,17 @@ async fn parse_python_imports_inner(
// Use cached content if available, otherwise fall back to deployed script
let code = match code_from_cache {
Some(content) => content,
None => {
sqlx::query_scalar!(
r#"
None => sqlx::query_scalar!(
r#"
SELECT content FROM script WHERE path = $1 AND workspace_id = $2
AND archived = false ORDER BY created_at DESC LIMIT 1
"#,
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string())
}
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string()),
};
if already_visited.contains(&rpath) {
backend/parsers/windmill-parser-py-imports/src/mapping.rs
+5
View File
@@ -383,5 +383,10 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! {
"vt" => "vt-py",
"grpc" => "grpcio",
"serpapi" => "google-search-results",
"pptx" => "python-pptx",
"OpenSSL" => "pyOpenSSL",
"win32api" => "pywin32",
"ruamel" => "ruamel.yaml",
"speech-recognition" => "SpeechRecognition",
// Add new entry here ^
};
backend/parsers/windmill-parser-py-imports/tests/tests.rs
+86 -1
View File
@@ -1,7 +1,9 @@
mod tests {
use sqlx::{Pool, Postgres};
use windmill_parser_py_imports::parse_python_imports;
use windmill_parser_py_imports::{
parse_code_for_imports, parse_python_imports, ImportPin, NImport,
};
#[sqlx::test(fixtures("base"))]
async fn test_parse_python_imports(db: Pool<Postgres>) -> anyhow::Result<()> {
@@ -117,4 +119,87 @@ def main():
Ok(())
}
fn auto(pkg: &str) -> NImport {
NImport::Auto { pkg: pkg.to_string(), key: None }
}
#[test]
fn test_parse_lazy_imports() -> anyhow::Result<()> {
let code = "
lazy import requests
lazy import numpy # pin: numpy==2.1.0
lazy from pandas import DataFrame
lazy from f.shared import helper
lazy = 1
def main():
pass
";
assert_eq!(
parse_code_for_imports(code, "f/foo/bar")?,
vec![
NImport::Pin {
pins: vec![ImportPin {
pkg: "numpy==2.1.0".to_string(),
path: "f/foo/bar".to_string()
}],
key: "numpy".to_string()
},
auto("pandas"),
auto("requests"),
NImport::Relative("f/shared".to_string()),
]
);
Ok(())
}
#[test]
fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> {
let code = r#"
import requests
type Alias = list[int]
NAME = "x"
D = {"a": 1}
L = [[1], [2]]
TEMPLATE = t"hello {NAME}"
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
PEP798 = [*a for a in L]
try:
import ujson
except ImportError, ValueError:
pass
def main():
pass
"#;
assert_eq!(
parse_code_for_imports(code, "f/foo/bar")?,
vec![auto("requests")]
);
Ok(())
}
#[test]
fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> {
for example in ["def main[T](x: T):", "def main(x):"] {
let code = format!(
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n"
);
assert_eq!(
parse_code_for_imports(&code, "f/foo/bar")?,
vec![NImport::Pin {
pins: vec![ImportPin {
pkg: "requests==2.32.0".to_string(),
path: "f/foo/bar".to_string()
}],
key: "requests".to_string()
}],
"{code}"
);
}
Ok(())
}
}
backend/parsers/windmill-parser-py/Cargo.toml
+2 -2
View File
@@ -10,8 +10,8 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
rustpython-parser.workspace = true
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
itertools.workspace = true
serde_json.workspace = true
anyhow.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
backend/parsers/windmill-parser-py/src/lib.rs
+206 -103
View File
@@ -13,13 +13,11 @@ use itertools::Itertools;
use serde_json::json;
use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ};
use rustpython_parser::{
ast::{
Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt,
StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite,
},
Parse,
use ruff_python_ast::{
Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign,
StmtClassDef, StmtFunctionDef,
};
use ruff_python_parser::parse_module;
pub mod pydantic_parser;
@@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool {
|| code.contains("import dataclasses")
}
/// What follows `def <name>` or `async def <name>` on a line opening that function's
/// parameters or type parameters.
fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> {
let rest = line
.strip_prefix("async def ")
.or_else(|| line.strip_prefix("def "))?
.strip_prefix(name)?;
(rest.starts_with('(') || rest.starts_with('[')).then_some(rest)
}
fn filter_non_main(code: &str, main_name: &str) -> String {
let def_main = format!("def {}(", main_name);
let async_def_main = format!("async def {}(", main_name);
let mut filtered_code = String::new();
let mut code_iter = code.split("\n");
let mut remaining: String = String::new();
while let Some(line) = code_iter.next() {
if line.starts_with(&async_def_main) {
filtered_code += &async_def_main;
remaining += line.strip_prefix(&async_def_main).unwrap();
remaining += "\n";
remaining += &code_iter.join("\n");
break;
} else if line.starts_with(&def_main) {
filtered_code += &def_main;
remaining += line.strip_prefix(&def_main).unwrap();
if let Some(rest) = strip_def(line, main_name) {
filtered_code += &line[..line.len() - rest.len()];
remaining += rest;
remaining += "\n";
remaining += &code_iter.join("\n");
break;
@@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String {
return String::new();
}
let mut chars = remaining.chars();
// PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters.
if remaining.starts_with('[') {
let mut open_brackets = 0;
for c in chars.by_ref() {
if c == '[' {
open_brackets += 1;
} else if c == ']' {
open_brackets -= 1;
}
filtered_code.push(c);
if open_brackets == 0 {
break;
}
}
}
if chars.next() != Some('(') {
return String::new();
}
filtered_code.push('(');
let mut open_parens = 1;
while let Some(c) = chars.next() {
@@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
let filtered_code = filter_relevant_statements(code);
let ast = match Suite::parse(&filtered_code, "main.py") {
Ok(ast) => ast,
let ast = match parse_module(&filtered_code) {
Ok(parsed) => parsed.into_suite(),
Err(_) => return CodeMetadata { enums, descriptions },
};
for stmt in ast {
match stmt {
Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => {
let is_enum = bases.iter().any(|base| {
Stmt::ClassDef(class_def) if has_enum => {
let is_enum = class_def.bases().iter().any(|base| {
matches!(base, Expr::Name(ExprName { id, .. })
if id == "Enum" || id == "IntEnum" || id == "StrEnum"
|| id == "Flag" || id == "IntFlag")
@@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
let mut values = Vec::new();
let mut members = HashMap::new();
let StmtClassDef { name, body, .. } = class_def;
for item in body {
if let Stmt::Assign(StmtAssign { targets, value, .. }) = item {
if let Some(Expr::Name(ExprName { id: target_name, .. })) =
targets.first()
{
if !target_name.starts_with('_') {
if let Expr::Constant(ExprConstant {
value: Constant::Str(val),
..
}) = value.as_ref()
{
if let Expr::StringLiteral(val) = value.as_ref() {
let val = val.value.to_str();
values.push(val.to_string());
members.insert(target_name.to_string(), val.to_string());
}
@@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
}
}
}
Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => {
if &func_name == main_name {
Stmt::FunctionDef(StmtFunctionDef {
name: func_name, body, is_async: false, ..
}) if has_docstring => {
if func_name.as_str() == main_name {
if let Some(Stmt::Expr(expr_stmt)) = body.first() {
if let Expr::Constant(ExprConstant {
value: Constant::Str(docstring),
..
}) = expr_stmt.value.as_ref()
{
descriptions = parse_docstring_args(docstring);
if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() {
descriptions = parse_docstring_args(docstring.value.to_str());
}
}
}
@@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap<String, String> {
descriptions
}
/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring
/// showing an example for instance. When several lines do, only the parsed module tells which
/// one is the function.
fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option<Parameters> {
code.split('\n')
.filter(|line| strip_def(line, main_name).is_some())
.nth(1)?;
parse_module(code)
.ok()?
.into_suite()
.into_iter()
.find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(*parameters)
}
_ => None,
})
}
/// skip_params is a micro optimization for when we just want to find the main
/// function without parsing all the params.
pub fn parse_python_signature(
@@ -310,41 +347,39 @@ pub fn parse_python_signature(
// - If no models: parse only the filtered main function
let (params, module) = if should_parse_for_models(code) {
// Parse full code once for both Pydantic detection and signature extraction
let ast = Suite::parse(code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(code)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
// Extract main function from full AST
let params = ast.iter().find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => {
Some(args.as_ref().clone())
}
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
if name == &main_name =>
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(args.as_ref().clone())
Some(parameters.as_ref().clone())
}
_ => None,
});
// Keep AST for Pydantic/dataclass detection
(params, Some(ast))
} else if let Some(params) = main_params_among_lookalikes(code, &main_name) {
(Some(params), None)
} else {
// No models detected - parse only the filtered main function
let filtered_code = filter_non_main(code, &main_name);
if filtered_code.is_empty() {
(None, None)
} else {
let ast = Suite::parse(&filtered_code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(&filtered_code)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
let params = ast.into_iter().find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => {
Some(*args)
}
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
if &name == &main_name =>
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(*args)
Some(*parameters)
}
_ => None,
});
@@ -378,8 +413,6 @@ pub fn parse_python_signature(
if !skip_params && params.is_some() {
let params = params.unwrap();
let def_arg_start = params.args.len() - params.defaults().count();
// Two-pass approach for lazy metadata extraction:
// Pass 1: Parse types without enum info to determine if metadata is needed
// Pass 2: Re-parse unknown types with metadata only if necessary
@@ -392,11 +425,11 @@ pub fn parse_python_signature(
.iter()
.enumerate()
.map(|(i, x)| {
let arg_name = x.as_arg().arg.to_string();
let arg_name = x.parameter.name.to_string();
let (typ, has_default) = x
.as_arg()
.parameter
.annotation
.as_ref()
.as_deref()
.map_or((Typ::Unknown, false), |e| {
parse_expr(e, &empty_enums, module_ref)
});
@@ -423,21 +456,16 @@ pub fn parse_python_signature(
.into_iter()
.map(|(i, arg_name, mut typ, mut has_default)| {
if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() {
if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() {
if let Some(annotation) = params.args[i].parameter.annotation.as_deref() {
(typ, has_default) =
parse_expr(annotation, &metadata.enums, module_ref);
}
}
let default = if i >= def_arg_start {
params
.defaults()
.nth(i - def_arg_start)
.map(|expr| to_value(expr, &metadata.enums))
.flatten()
} else {
None
};
let default = params.args[i]
.default
.as_deref()
.and_then(|expr| to_value(expr, &metadata.enums));
let should_get_type_from_default = match &typ {
Typ::Unknown => true,
@@ -465,10 +493,10 @@ pub fn parse_python_signature(
// debounce accumulation support. Falls back to docstring
// description if no union array pattern is found.
let union_otyp = params.args[i]
.as_arg()
.parameter
.annotation
.as_ref()
.and_then(|ann| detect_py_union_array_otyp(ann.as_ref()));
.as_deref()
.and_then(detect_py_union_array_otyp);
Arg {
otyp: union_otyp.or_else(|| {
@@ -509,12 +537,8 @@ pub fn parse_python_signature(
}
}
fn parse_expr(
e: &Box<Expr>,
enums: &HashMap<String, EnumInfo>,
module: Option<&[Stmt]>,
) -> (Typ, bool) {
match e.as_ref() {
fn parse_expr(e: &Expr, enums: &HashMap<String, EnumInfo>, module: Option<&[Stmt]>) -> (Typ, bool) {
match e {
Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false),
Expr::Attribute(x) => {
if let Some(name) = x.value.as_name_expr() {
@@ -531,12 +555,9 @@ fn parse_expr(
}
}
Expr::BinOp(x) => {
if matches!(
x.right.as_ref(),
Expr::Constant(ExprConstant { value: Constant::None, .. })
) {
if x.right.is_none_literal_expr() {
(parse_expr(&x.left, enums, module).0, true)
} else if detect_py_union_array_otyp(e.as_ref()).is_some() {
} else if detect_py_union_array_otyp(e).is_some() {
// T | list[T] — parsed type is Unknown; otyp is set separately
(Typ::Unknown, false)
} else {
@@ -552,7 +573,7 @@ fn parse_expr(
.elts
.iter()
.map(|x| match x {
Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()),
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
.filter_map(|x| x)
@@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String {
}
}
fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
fn to_value(et: &Expr, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
match et {
Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)),
Expr::Dict(ExprDict { keys, values, .. }) => {
let v = keys
.into_iter()
.zip(values)
.map(|(k, v)| {
let key = k
Expr::NoneLiteral(_) => Some(json!(null)),
Expr::BooleanLiteral(b) => Some(json!(b.value)),
Expr::StringLiteral(s) => Some(json!(s.value.to_str())),
Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::<Vec<u8>>())),
Expr::NumberLiteral(n) => Some(match &n.value {
Number::Int(i) => {
serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number"))
}
Number::Float(f) => json!(f),
Number::Complex { real, imag } => json!([real, imag]),
}),
Expr::EllipsisLiteral(_) => Some(json!("...")),
Expr::Dict(ExprDict { items, .. }) => {
let v = items
.iter()
.map(|item| {
let key = item
.key
.as_ref()
.map(|e| to_value(e, enums))
.flatten()
@@ -642,7 +674,7 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
_ => None,
})
.unwrap_or_else(|| "no_key".to_string());
(key, to_value(&v, enums))
(key, to_value(&item.value, enums))
})
.collect::<HashMap<String, _>>();
Some(json!(v))
@@ -672,20 +704,6 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
}
}
fn constant_to_value(c: &Constant) -> serde_json::Value {
match c {
Constant::None => json!(null),
Constant::Bool(b) => json!(b),
Constant::Str(s) => json!(s),
Constant::Bytes(b) => json!(b),
Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")),
Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::<Vec<_>>()),
Constant::Float(f) => json!(f),
Constant::Complex { real, imag } => json!([real, imag]),
Constant::Ellipsis => json!("..."),
}
}
#[cfg(test)]
mod tests {
@@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED):
);
Ok(())
}
#[test]
fn test_parse_python_sig_type_params() -> anyhow::Result<()> {
for code in [
"def main[T](x: str, y: int = 3):\n return x\n",
"async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n",
// parsed whole rather than pre-filtered down to `main`
"from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n",
] {
let result = parse_python_signature(code, None, false)?;
assert_eq!(result.auto_kind, None, "{code}");
assert_eq!(
result
.args
.iter()
.map(|a| (a.name.as_str(), &a.typ, &a.default))
.collect::<Vec<_>>(),
vec![
("x", &Typ::Str(None), &None),
("y", &Typ::Int, &Some(json!(3)))
],
"{code}"
);
}
Ok(())
}
// A script with a model class is parsed whole, so syntax anywhere in it reaches the parser.
#[test]
fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> {
let code = r#"
from dataclasses import dataclass
lazy import requests
lazy from pandas import DataFrame
type Alias = list[int]
@dataclass
class Point:
x: int
try:
import a
except ImportError, ValueError:
pass
D = {"a": 1}
L = [[1], [2]]
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
PEP798 = [*a for a in L]
def main(p: Point, name: str = "world"):
return t"hello {name}"
"#;
let result = parse_python_signature(code, None, false)?;
assert_eq!(result.args.len(), 2);
assert_eq!(
result.args[0].typ,
Typ::Object(ObjectType::new(
Some("Point".to_string()),
Some(vec![windmill_parser::ObjectProperty {
key: "x".to_string(),
typ: Box::new(Typ::Int)
}])
))
);
assert_eq!(result.args[1].default, Some(json!("world")));
Ok(())
}
#[test]
fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> {
for example in ["def main[T](x: T):", "def main(x: int, y: int):"] {
let code = format!(
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n"
);
let result = parse_python_signature(&code, None, false)?;
assert_eq!(
result.args.iter().map(|a| &a.name).collect::<Vec<_>>(),
vec!["value"],
"{code}"
);
}
Ok(())
}
}
backend/parsers/windmill-parser-py/src/pydantic_parser.rs
+9 -9
View File
@@ -11,8 +11,8 @@
//! This module provides functionality to detect and parse Pydantic models and Python
//! dataclasses from Python AST, enabling automatic UI generation for complex data structures.
use rustpython_parser::ast::{
Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign,
use ruff_python_ast::{
Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign,
};
use std::collections::HashSet;
use windmill_parser::{ObjectProperty, ObjectType, Typ};
@@ -64,7 +64,7 @@ fn detect_model_type_impl(
// Mark as being visited
visited.insert(class_name.to_string());
let result = if is_pydantic_base(&class_def.bases) {
let result = if is_pydantic_base(class_def.bases()) {
// Pydantic BaseModel
parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited)
} else if has_dataclass_decorator(&class_def.decorator_list)
@@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool {
}
/// Checks if a class has @dataclass decorator (standard library)
fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
fn has_dataclass_decorator(decorators: &[Decorator]) -> bool {
for decorator in decorators {
match decorator {
match &decorator.expression {
// Match: @dataclass
Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => {
return true;
@@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
}
/// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2)
fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool {
fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool {
for decorator in decorators {
match decorator {
match &decorator.expression {
// Match: @pydantic.dataclasses.dataclass
Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => {
if let Expr::Attribute(ExprAttribute {
@@ -350,10 +350,10 @@ fn extract_field_type(
// String annotations: "ForwardRef" (forward references)
// Not fully supported - return Unknown with warning
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
Expr::StringLiteral(s) => {
eprintln!(
"Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown",
forward_ref = s,
forward_ref = s.value.to_str(),
);
Typ::Unknown
}
backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs
+10 -7
View File
@@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets(
Ok(mut sql_assets) => {
if let Some(schema) = schema {
for asset in &mut sql_assets.assets {
if asset.kind == kind && asset.path.starts_with(asset_name) {
asset.path = format!(
"{}/{}.{}",
asset_name,
schema,
&asset.path[asset_name.len() + 1..]
);
if asset.kind != kind {
continue;
}
// A statement can name the database alone, with no table to qualify.
let table = asset
.path
.strip_prefix(asset_name)
.and_then(|rest| rest.strip_prefix('/'));
if let Some(table) = table {
asset.path = format!("{asset_name}/{schema}.{table}");
}
}
}
backend/parsers/windmill-parser-sql/src/lib.rs
+44 -1
View File
@@ -235,7 +235,7 @@ lazy_static::lazy_static! {
// -- $name (type) = default
static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \(([A-Za-z0-9_\[\]]+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap();
@@ -254,6 +254,12 @@ fn parsed_default(parsed_typ: &Typ, default: String) -> Option<serde_json::Value
Typ::Str(_) if default.len() >= 2 && default.starts_with("'") && default.ends_with("'") => {
Some(json!(&default[1..default.len() - 1]))
}
Typ::List(_) => Some(
serde_json::from_str::<serde_json::Value>(&default)
.ok()
.filter(|v| v.is_array())
.unwrap_or_else(|| json!(default)),
),
_ => Some(json!(default)),
}
}
@@ -1039,7 +1045,11 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ {
}
pub fn parse_snowflake_typ(typ: &str) -> Typ {
if let Some(base_typ) = typ.strip_suffix("[]") {
return Typ::List(Box::new(parse_snowflake_typ(base_typ)));
}
match typ {
"array" => Typ::List(Box::new(Typ::Unknown)),
"varchar" | "text" | "string" | "char" | "character" => Typ::Str(None),
"binary" | "varbinary" => Typ::Bytes,
"date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz"
@@ -1591,6 +1601,39 @@ SELECT ?;
Ok(())
}
#[test]
fn test_parse_snowflake_array_types() -> anyhow::Result<()> {
let code = r#"
-- ? groups (varchar[])
-- ? ids (int[]) = [1, 2]
-- ? anything (ARRAY)
SELECT 1;
"#;
let args = parse_snowflake_sig(code)?.args;
assert_eq!(args[1].default, Some(json!([1, 2])));
let got = args
.iter()
.map(|a| (a.name.as_str(), a.otyp.as_deref(), &a.typ))
.collect::<Vec<_>>();
assert_eq!(
got,
vec![
(
"groups",
Some("varchar[]"),
&Typ::List(Box::new(Typ::Str(None)))
),
("ids", Some("int[]"), &Typ::List(Box::new(Typ::Int))),
(
"anything",
Some("array"),
&Typ::List(Box::new(Typ::Unknown))
),
]
);
Ok(())
}
#[test]
fn test_parse_mssql_sig() -> anyhow::Result<()> {
let code = r#"
backend/parsers/windmill-parser-wac/Cargo.toml
+3 -2
View File
@@ -9,8 +9,9 @@ name = "windmill_parser_wac"
path = "./src/lib.rs"
[dependencies]
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
ruff_text_size.workspace = true
swc_common.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
backend/parsers/windmill-parser-wac/src/python.rs
+135 -190
View File
@@ -1,12 +1,12 @@
use std::collections::HashMap;
use rustpython_parser::{
ast::{
Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry,
StmtTryStar, StmtWhile,
},
Parse,
use ruff_python_ast::{
CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName,
Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry,
StmtWhile,
};
use ruff_python_parser::parse_module;
use ruff_text_size::{Ranged, TextSize};
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
use crate::validation::{self, CompileError};
@@ -42,9 +42,9 @@ type TaskFunctions = HashMap<String, Option<String>>;
fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
let mut tasks = HashMap::new();
for stmt in stmts {
if let Stmt::AsyncFunctionDef(func) = stmt {
if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt {
for dec in &func.decorator_list {
match dec {
match &dec.expression {
// @task (bare decorator)
Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => {
tasks.insert(func.name.to_string(), None);
@@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
if id.as_str() == "task_script" || id.as_str() == "task_flow" {
// Extract the path from the first positional argument
let path = call.args.first().and_then(|arg| {
if let Expr::Constant(c) = arg {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
}
None
let path = call.arguments.args.first().and_then(|arg| match arg {
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
});
// Extract variable name from target
if let Some(Expr::Name(ExprName { id: var_name, .. })) =
@@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
/// Extract the `path=` keyword argument from a `@task(path="...")` call.
fn extract_task_path_kwarg(call: &ExprCall) -> Option<String> {
for kw in &call.keywords {
for kw in &call.arguments.keywords {
if let Some(ref arg) = kw.arg {
if arg.as_str() == "path" {
if let Expr::Constant(c) = &kw.value {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
if let Expr::StringLiteral(s) = &kw.value {
return Some(s.value.to_str().to_string());
}
}
}
@@ -150,10 +144,9 @@ impl WacWalker {
fn line_of_expr(&self, expr: &Expr) -> usize {
let offset = match expr {
Expr::Call(c) => c.range.start().to_usize(),
Expr::Await(a) => a.range.start().to_usize(),
Expr::Attribute(a) => a.range.start().to_usize(),
Expr::Name(n) => n.range.start().to_usize(),
Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => {
expr.start().to_usize()
}
_ => 0,
};
self.line_index.line_of(offset)
@@ -161,17 +154,15 @@ impl WacWalker {
fn line_of_stmt(&self, stmt: &Stmt) -> usize {
let offset = match stmt {
Stmt::If(s) => s.range.start().to_usize(),
Stmt::For(s) => s.range.start().to_usize(),
Stmt::While(s) => s.range.start().to_usize(),
Stmt::Return(s) => s.range.start().to_usize(),
Stmt::Expr(s) => s.range.start().to_usize(),
Stmt::Try(s) => s.range.start().to_usize(),
Stmt::TryStar(s) => s.range.start().to_usize(),
Stmt::Assign(s) => s.range.start().to_usize(),
Stmt::AnnAssign(s) => s.range.start().to_usize(),
Stmt::FunctionDef(s) => s.range.start().to_usize(),
Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(),
Stmt::If(_)
| Stmt::For(_)
| Stmt::While(_)
| Stmt::Return(_)
| Stmt::Expr(_)
| Stmt::Try(_)
| Stmt::Assign(_)
| Stmt::AnnAssign(_)
| Stmt::FunctionDef(_) => stmt.start().to_usize(),
_ => 0,
};
self.line_index.line_of(offset)
@@ -190,9 +181,7 @@ impl WacWalker {
/// Check if an expression is `asyncio.gather(...)` call
fn is_asyncio_gather_call(expr: &Expr) -> bool {
if let Expr::Call(call) = expr {
if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) =
call.func.as_ref()
{
if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() {
if attr.as_str() == "gather" {
if let Expr::Name(ExprName { id, .. }) = value.as_ref() {
return id.as_str() == "asyncio";
@@ -222,20 +211,20 @@ impl WacWalker {
fn expr_to_source(expr: &Expr) -> String {
match expr {
Expr::Compare(c) => {
let left = Self::expr_to_source(&c.left);
if let Some(comparator) = c.comparators.first() {
let left = Self::expr_to_source(&c.operands[0]);
if let Some(comparator) = c.comparators().first() {
let right = Self::expr_to_source(comparator);
let op = match c.ops.first() {
Some(rustpython_parser::ast::CmpOp::Gt) => ">",
Some(rustpython_parser::ast::CmpOp::Lt) => "<",
Some(rustpython_parser::ast::CmpOp::GtE) => ">=",
Some(rustpython_parser::ast::CmpOp::LtE) => "<=",
Some(rustpython_parser::ast::CmpOp::Eq) => "==",
Some(rustpython_parser::ast::CmpOp::NotEq) => "!=",
Some(rustpython_parser::ast::CmpOp::In) => "in",
Some(rustpython_parser::ast::CmpOp::NotIn) => "not in",
Some(rustpython_parser::ast::CmpOp::Is) => "is",
Some(rustpython_parser::ast::CmpOp::IsNot) => "is not",
Some(CmpOp::Gt) => ">",
Some(CmpOp::Lt) => "<",
Some(CmpOp::GtE) => ">=",
Some(CmpOp::LtE) => "<=",
Some(CmpOp::Eq) => "==",
Some(CmpOp::NotEq) => "!=",
Some(CmpOp::In) => "in",
Some(CmpOp::NotIn) => "not in",
Some(CmpOp::Is) => "is",
Some(CmpOp::IsNot) => "is not",
None => "?",
};
format!("{left} {op} {right}")
@@ -253,14 +242,14 @@ impl WacWalker {
format!("{value}.{}", a.attr)
}
Expr::Name(n) => n.id.to_string(),
Expr::Constant(c) => match &c.value {
rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""),
rustpython_parser::ast::Constant::Int(i) => i.to_string(),
rustpython_parser::ast::Constant::Float(f) => f.to_string(),
rustpython_parser::ast::Constant::Bool(b) => b.to_string(),
rustpython_parser::ast::Constant::None => "None".to_string(),
_ => "...".to_string(),
Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()),
Expr::NumberLiteral(n) => match &n.value {
Number::Int(i) => i.to_string(),
Number::Float(f) => f.to_string(),
Number::Complex { .. } => "...".to_string(),
},
Expr::BooleanLiteral(b) => b.value.to_string(),
Expr::NoneLiteral(_) => "None".to_string(),
_ => "...".to_string(),
}
}
@@ -279,8 +268,16 @@ impl WacWalker {
match stmt {
Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value),
Stmt::Assign(a) => self.expr_contains_step(&a.value),
Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::If(s) => {
self.body_contains_step(&s.body)
|| s.elif_else_clauses
.iter()
.any(|clause| self.body_contains_step(&clause.body))
}
// `async for` is not modelled: its body is skipped like any unsupported statement
Stmt::For(s) if !s.is_async => {
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
}
Stmt::While(s) => {
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
}
@@ -289,19 +286,7 @@ impl WacWalker {
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
})
}
Stmt::TryStar(s) => {
self.body_contains_step(&s.body)
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
})
}
Stmt::Return(_) => false,
@@ -319,11 +304,12 @@ impl WacWalker {
match expr {
Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value),
Expr::Call(call) => {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
return true;
}
if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) {
return call.args.iter().any(|a| self.expr_contains_step(a));
if Self::is_asyncio_gather_call(expr) {
return call
.arguments
.args
.iter()
.any(|a| self.expr_contains_step(a));
}
false
}
@@ -370,12 +356,11 @@ impl WacWalker {
Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value),
Stmt::Assign(a) => self.walk_expr_stmt(&a.value),
Stmt::If(if_stmt) => self.walk_if(if_stmt),
Stmt::For(for_stmt) => self.walk_for(for_stmt),
Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt),
Stmt::While(while_stmt) => self.walk_while(while_stmt),
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt),
Stmt::Return(ret) => self.walk_return(ret),
Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => {
Stmt::FunctionDef(_) => {
if self.stmt_contains_step(stmt) {
self.errors.push(validation::error_step_in_nested_function(
self.line_of_stmt(stmt),
@@ -392,7 +377,7 @@ impl WacWalker {
if let Expr::Await(ExprAwait { value, .. }) = expr {
// await task_fn(...)
if let Expr::Call(call) = value.as_ref() {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
if self.is_task_fn_call(value) {
return self.emit_step(call, expr);
}
// Check for SDK-level calls: step(), sleep(), wait_for_approval()
@@ -430,15 +415,12 @@ impl WacWalker {
"step" => {
// step("name", fn) — extract the name from the first string argument
let name = call
.arguments
.args
.first()
.and_then(|arg| {
if let Expr::Constant(c) = arg {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
}
None
.and_then(|arg| match arg {
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
.unwrap_or_else(|| "step".to_string());
let id = self.next_id();
@@ -452,6 +434,7 @@ impl WacWalker {
}
"sleep" => {
let seconds = call
.arguments
.args
.first()
.map(|arg| Self::expr_to_source(arg))
@@ -517,10 +500,10 @@ impl WacWalker {
});
let mut step_ids = Vec::new();
for arg in &gather_call.args {
for arg in &gather_call.arguments.args {
// Each arg should be task_fn(...)
if let Expr::Call(call) = arg {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
if self.is_task_fn_call(arg) {
let (name, script) = self
.extract_step_info_from_task_call(call)
.unwrap_or(("unknown".into(), "unknown".into()));
@@ -553,15 +536,33 @@ impl WacWalker {
}
fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> {
let has_steps_in_body = self.body_contains_step(&if_stmt.body);
let has_steps_in_else = self.body_contains_step(&if_stmt.orelse);
self.walk_if_chain(
if_stmt.start(),
&if_stmt.test,
&if_stmt.body,
&if_stmt.elif_else_clauses,
)
}
/// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it.
fn walk_if_chain(
&mut self,
start: TextSize,
test: &Expr,
body: &[Stmt],
clauses: &[ElifElseClause],
) -> Option<(String, String)> {
let has_steps_in_body = self.body_contains_step(body);
let has_steps_in_else = clauses
.iter()
.any(|clause| self.body_contains_step(&clause.body));
if !has_steps_in_body && !has_steps_in_else {
return None;
}
let line = self.line_index.line_of(if_stmt.range.start().to_usize());
let condition_source = Self::expr_to_source(&if_stmt.test);
let line = self.line_index.line_of(start.to_usize());
let condition_source = Self::expr_to_source(test);
let branch_id = self.next_id();
let branch_node_id = self.add_node(DagNode {
@@ -573,15 +574,19 @@ impl WacWalker {
let mut last_ids = Vec::new();
if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) {
if let Some((true_first, true_last)) = self.walk_body(body) {
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
last_ids.push(true_last);
} else {
last_ids.push(branch_node_id.clone());
}
if !if_stmt.orelse.is_empty() {
if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) {
if let Some((clause, clauses)) = clauses.split_first() {
let orelse = match &clause.test {
Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses),
None => self.walk_body(&clause.body),
};
if let Some((else_first, else_last)) = orelse {
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
last_ids.push(else_last);
} else {
@@ -611,7 +616,7 @@ impl WacWalker {
return None;
}
let line = self.line_index.line_of(for_stmt.range.start().to_usize());
let line = self.line_index.line_of(for_stmt.start().to_usize());
let iter_source = Self::expr_to_source(&for_stmt.iter);
let start_id = self.next_id();
@@ -644,7 +649,7 @@ impl WacWalker {
return None;
}
let line = self.line_index.line_of(while_stmt.range.start().to_usize());
let line = self.line_index.line_of(while_stmt.start().to_usize());
let condition = Self::expr_to_source(&while_stmt.test);
let start_id = self.next_id();
@@ -677,39 +682,14 @@ impl WacWalker {
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
});
if !has_steps {
return None;
}
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
self.emit_try_catch_branch(
&try_stmt.body,
&try_stmt.handlers,
&try_stmt.finalbody,
line,
)
}
fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> {
let has_steps = self.body_contains_step(&try_stmt.body)
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
});
if !has_steps {
return None;
}
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
let line = self.line_index.line_of(try_stmt.start().to_usize());
self.emit_try_catch_branch(
&try_stmt.body,
&try_stmt.handlers,
@@ -721,7 +701,7 @@ impl WacWalker {
fn emit_try_catch_branch(
&mut self,
try_body: &[Stmt],
handlers: &[rustpython_parser::ast::ExceptHandler],
handlers: &[ExceptHandler],
finally_body: &[Stmt],
line: usize,
) -> Option<(String, String)> {
@@ -746,7 +726,7 @@ impl WacWalker {
// Except handlers
for handler in handlers {
match handler {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
ExceptHandler::ExceptHandler(eh) => {
if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) {
self.add_edge(&branch_node_id, &catch_first, Some("except".to_string()));
last_ids.push(catch_last);
@@ -783,7 +763,7 @@ impl WacWalker {
}
fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> {
let line = self.line_index.line_of(ret.range.start().to_usize());
let line = self.line_index.line_of(ret.start().to_usize());
let id = self.next_id();
let node_id = self.add_node(DagNode {
id: id.clone(),
@@ -796,12 +776,12 @@ impl WacWalker {
}
/// Extract workflow function parameters (no longer skips ctx)
fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
fn extract_params(args: &Parameters) -> Vec<Param> {
let mut params = Vec::new();
for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) {
let name = arg_with_default.def.arg.to_string();
let name = arg_with_default.parameter.name.to_string();
let typ = arg_with_default
.def
.parameter
.annotation
.as_ref()
.map(|ann| WacWalker::expr_to_source(ann));
@@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
}
pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
let ast = rustpython_parser::ast::Suite::parse(code, "<workflow>")
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?;
let ast = parse_module(code)
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?
.into_suite();
// First pass: collect @task functions
let task_functions = collect_task_functions(&ast);
// Find the @workflow async def
let workflow_fn = ast.iter().find_map(|stmt| {
if let Stmt::AsyncFunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return Some(func);
let workflow_fns = || {
ast.iter().filter_map(|stmt| match stmt {
Stmt::FunctionDef(func)
if func.decorator_list.iter().any(|dec| {
matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow")
}) =>
{
Some(func)
}
}
// Also check non-async for error reporting
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return None; // Will be reported as not-async below
}
}
None
});
_ => None,
})
};
// Check for non-async workflow function
let non_async_workflow = ast.iter().find_map(|stmt| {
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
let line_index = LineIndex::new(code);
return Some(line_index.line_of(func.range.start().to_usize()));
}
}
None
});
// Find the @workflow async def
let workflow_fn = workflow_fns().find(|func| func.is_async);
// Check for non-async workflow function. The statement starts at its first decorator,
// the name sits on the `def` line the error points at.
let non_async_workflow = workflow_fns()
.find(|func| !func.is_async)
.map(|func| LineIndex::new(code).line_of(func.name.start().to_usize()));
if let Some(line) = non_async_workflow {
if workflow_fn.is_none() {
@@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError
vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }]
})?;
let params = extract_params(&workflow_fn.args);
let params = extract_params(&workflow_fn.parameters);
let source_hash = compute_source_hash(code);
let mut walker = WacWalker::new(code, task_functions);
@@ -894,13 +849,3 @@ fn compute_source_hash(code: &str) -> String {
hasher.update(code.as_bytes());
format!("{:x}", hasher.finalize())
}
trait ToUsize {
fn to_usize(self) -> usize;
}
impl ToUsize for rustpython_parser::text_size::TextSize {
fn to_usize(self) -> usize {
u32::from(self) as usize
}
}
backend/parsers/windmill-parser-wac/tests/python_tests.rs
+74
View File
@@ -363,3 +363,77 @@ async def main(x: str):
));
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return));
}
#[test]
fn test_elif_chain_nests_branches() {
let code = r#"
from wmill import workflow, task
@task
async def a(): ...
@task
async def b(): ...
@task
async def c(): ...
@workflow
async def main(x: int):
if x > 2:
await a()
elif x == 2:
await b()
else:
await c()
"#;
let dag = parse_python_workflow(code).expect("should parse");
let branches: Vec<(&str, usize)> = dag
.nodes
.iter()
.filter_map(|n| match &n.node_type {
DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)),
_ => None,
})
.collect();
assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]);
// The `elif` branch hangs off the `false` edge of the `if` before it
let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id);
assert!(dag
.edges
.iter()
.any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false")));
assert_eq!(
dag.nodes
.iter()
.filter(|n| matches!(n.node_type, DagNodeType::Merge))
.count(),
2
);
}
#[test]
fn test_workflow_with_recent_syntax() {
let code = r#"
from wmill import workflow, task
lazy import polars
type Rows = list[dict]
@task
async def load[T](rows: T): ...
@workflow
async def main(name: str):
try:
await load(t"hello {name}")
except ValueError, TypeError:
pass
"#;
let dag = parse_python_workflow(code).expect("should parse");
assert!(dag
.nodes
.iter()
.any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load")));
}
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+455 -325
View File
File diff suppressed because it is too large Load Diff
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.826.0"
version = "1.830.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/src/main.rs
+94 -260
View File
@@ -63,12 +63,14 @@ use windmill_common::{
SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING,
SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING,
SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING,
SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING,
TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING,
WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING,
},
hub_resource_types,
scripts::ScriptLang,
stats_oss::schedule_stats,
triggers::TriggerKind,
@@ -121,7 +123,8 @@ use windmill_worker::{
BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR,
POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
PY314_CACHE_DIR, PY315_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR,
TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
@@ -401,111 +404,16 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
Ok(())
}
/// Raw resource type from hub API (schema is a JSON string)
#[derive(serde::Deserialize)]
struct HubResourceTypeRaw {
pub id: i64,
pub name: String,
pub schema: Option<String>,
pub app: String,
pub description: Option<String>,
/// Absent from hubs predating the column, and from caches written before it.
#[serde(default)]
pub format_extension: Option<String>,
/// Doubly optional, so a hub predating the field (no key) is told apart from a type the
/// hub leaves unnamed (null).
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option"
)]
pub display_name: Option<Option<String>>,
}
/// Processed resource type with parsed schema
#[derive(serde::Deserialize, serde::Serialize, Clone)]
pub struct HubResourceType {
pub id: i64,
pub name: String,
pub schema: Option<serde_json::Value>,
pub app: String,
pub description: Option<String>,
/// Doubly optional on purpose. A cache written before this column has no key at
/// all (`None`) and must leave the stored extension alone; one written since
/// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely
/// dropping it and must clear. A single `Option` conflates the two, and picking
/// either meaning breaks the other — as does plain serde, which folds `null`
/// into the outer `None`, hence the wrapping deserializer.
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option",
skip_serializing_if = "Option::is_none"
)]
pub format_extension: Option<Option<String>>,
/// Doubly optional like `format_extension`: a cache written before the field leaves the
/// stored name alone, while a null from the hub clears it.
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option",
skip_serializing_if = "Option::is_none"
)]
pub display_name: Option<Option<String>>,
}
const HUB_RT_CACHE_FILE: &str = "resource_types.json";
async fn cache_hub_resource_types() -> anyhow::Result<()> {
println!("Caching resource types from hub...");
let response = HTTP_CLIENT
.get(format!("{}/resource_types/list", DEFAULT_HUB_BASE_URL))
.header("Accept", "application/json")
.send()
.await
.with_context(|| "Failed to fetch resource types from hub")?;
if !response.status().is_success() {
anyhow::bail!(
"Failed to fetch resource types from hub: {}",
response.status()
);
}
let raw_types: Vec<HubResourceTypeRaw> = response
.json::<Vec<HubResourceTypeRaw>>()
.await
.with_context(|| "Failed to parse resource types from hub")?;
// Parse schema strings into JSON values
let resource_types: Vec<HubResourceType> = raw_types
.into_iter()
.filter_map(|rt| {
let schema = match rt.schema {
Some(s) => match serde_json::from_str(&s) {
Ok(v) => Some(v),
Err(e) => {
println!("Warning: failed to parse schema for {}: {}", rt.name, e);
return None;
}
},
None => None,
};
Some(HubResourceType {
id: rt.id,
name: rt.name,
schema,
app: rt.app,
description: rt.description,
format_extension: Some(rt.format_extension),
display_name: rt.display_name,
})
})
.collect();
let resource_types = hub_resource_types::fetch(DEFAULT_HUB_BASE_URL, None, None).await?;
println!("Fetched {} resource types from hub", resource_types.len());
create_dir_all(&*HUB_RT_CACHE_DIR)?;
let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE);
let cache_path = hub_resource_types::cache_path();
let content = serde_json::to_string_pretty(&resource_types)
.with_context(|| "Failed to serialize resource types")?;
@@ -516,147 +424,6 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> {
Ok(())
}
pub async fn sync_cached_resource_types(db: &sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE);
if tokio::fs::metadata(&cache_path).await.is_err() {
tracing::info!(
"No cached resource types found at {}, skipping sync",
cache_path
);
return Ok(());
}
tracing::info!("Syncing cached resource types to admins workspace...");
let content = tokio::fs::read_to_string(&cache_path)
.await
.with_context(|| format!("Failed to read cache file from {}", cache_path))?;
let cached_types: Vec<HubResourceType> =
serde_json::from_str(&content).with_context(|| "Failed to parse cached resource types")?;
tracing::info!("Found {} cached resource types", cached_types.len());
// Get existing resource types in admins workspace. `format_extension` is part of
// the comparison below, so a type whose only change is gaining or losing it is
// not mistaken for unchanged; `is_fileset` decides whether it may take one.
let existing_types: Vec<(
String,
Option<serde_json::Value>,
Option<String>,
Option<String>,
bool,
Option<String>,
)> = sqlx::query_as(
"SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'",
)
.fetch_all(db)
.await
.with_context(|| "Failed to fetch existing resource types")?;
let existing_map: std::collections::HashMap<
String,
(
Option<serde_json::Value>,
Option<String>,
Option<String>,
bool,
Option<String>,
),
> = existing_types
.into_iter()
.map(
|(name, schema, desc, format_extension, is_fileset, display_name)| {
(
name,
(schema, desc, format_extension, is_fileset, display_name),
)
},
)
.collect();
let mut synced_count = 0;
let mut skipped_count = 0;
for rt in cached_types {
let existing = existing_map.get(&rt.name);
let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false);
let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone());
let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone());
// A fileset is a set of files, so it cannot also be one file. Create, update
// and the manual sync all reject the pair; this writer would otherwise
// persist it onto a same-named local fileset.
//
// A cache with no key at all leaves the stored value alone, so the target is
// what is already there — which is also what makes the comparison below
// agree with the write instead of re-upserting the row on every boot.
let format_extension = if is_fileset {
None
} else {
match &rt.format_extension {
Some(from_cache) => from_cache.clone(),
None => stored_extension.clone(),
}
};
// No key in the cache leaves the stored name alone, as for the extension. So does a name
// too long for the column: one bad entry must not fail the upsert and end the sync.
let display_name = match &rt.display_name {
Some(Some(name)) if name.chars().count() > 100 => {
tracing::warn!(
"Ignoring the display_name of resource type {}: longer than 100 characters",
rt.name
);
stored_display_name.clone()
}
Some(from_cache) => from_cache.clone(),
None => stored_display_name.clone(),
};
if let Some((existing_schema, existing_desc, _, _, _)) = existing {
if existing_schema == &rt.schema
&& existing_desc == &rt.description
&& stored_extension == format_extension
&& stored_display_name == display_name
{
skipped_count += 1;
continue;
}
}
// Insert or update resource type
sqlx::query(
// `format_extension` is resolved above rather than coalesced here: a
// COALESCE could never clear one, so a hub that dropped an extension
// would leave the stale value behind forever.
"INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)
VALUES ('admins', $1, $2, $3, $4, $5, now())
ON CONFLICT (workspace_id, name) DO UPDATE
SET schema = EXCLUDED.schema, description = EXCLUDED.description,
format_extension = EXCLUDED.format_extension,
display_name = EXCLUDED.display_name, edited_at = now()",
)
.bind(&rt.name)
.bind(&rt.schema)
.bind(&rt.description)
.bind(&format_extension)
.bind(&display_name)
.execute(db)
.await
.with_context(|| format!("Failed to upsert resource type {}", rt.name))?;
synced_count += 1;
}
tracing::info!(
"Synced {} resource types to admins workspace ({} skipped as unchanged)",
synced_count,
skipped_count
);
Ok(())
}
fn print_help() {
println!("Windmill - a fast, open-source workflow engine and job runner.");
println!();
@@ -670,6 +437,8 @@ fn print_help() {
println!(" cache-rt Pre-cache hub resource types");
println!(" sync-config <file> Sync instance config from a YAML file to the database");
println!(" operator Run the Kubernetes operator (watches a ConfigMap)");
#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))]
println!(" oidc-signer <conformance|test-signer> (ee) Check an external OIDC signer, or run a file-backed one for tests");
println!();
println!("Environment variables (name = default):");
println!(" DATABASE_URL = <required> The Postgres database url.");
@@ -696,7 +465,7 @@ fn print_help() {
println!(" LICENSE_KEY = None (EE only) Enterprise license key (workers require valid key)");
println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup");
println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool");
println!(" SYNC_CACHED_RT = false Sync cached resource types to admins workspace on server start");
println!(" SYNC_CACHED_RT = false Sync the image's cached resource types to the admins workspace on server start");
println!(" HUB_BASE_URL = https://hub.windmill.dev Hub to fetch scripts from in `cache` mode (server/worker use the DB setting instead)");
println!();
println!("Notes:");
@@ -704,6 +473,38 @@ fn print_help() {
println!("- At startup, Windmill logs currently set configuration keys for visibility.");
}
async fn otel_environment(conn: &Connection) -> String {
if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") {
environment
} else {
load_base_url(conn)
.await
.unwrap_or_else(|_| "local".to_string())
.trim_start_matches("https://")
.trim_start_matches("http://")
.split(".")
.next()
.unwrap_or_else(|| "local")
.to_string()
}
}
/// Tracing for a subcommand that writes instance config. It never runs the server startup
/// that loads the OTel settings, and its audit events are exported under them.
///
/// The subscriber it installs depends on those settings, so `early_logs` is what prints
/// the events of connecting and loading them; it gives way once the real one is in.
async fn init_declarative_writer_tracing(
db: &windmill_common::DB,
service: &str,
early_logs: tracing::subscriber::DefaultGuard,
) {
load_otel(db).await;
let environment = otel_environment(&Connection::Sql(db.clone())).await;
drop(early_logs);
windmill_common::tracing_init::initialize_subcommand_tracing(service, &HOSTNAME, &environment);
}
async fn windmill_main() -> anyhow::Result<()> {
let (killpill_tx, mut killpill_rx) = KillpillSender::new(2);
let mut monitor_killpill_rx = killpill_tx.subscribe();
@@ -807,7 +608,6 @@ async fn windmill_main() -> anyhow::Result<()> {
return Ok(());
}
"sync-config" => {
tracing_subscriber::fmt::init();
let path = std::env::args().nth(2).unwrap_or_else(|| {
eprintln!("Usage: windmill sync-config <file>");
std::process::exit(1);
@@ -821,15 +621,40 @@ async fn windmill_main() -> anyhow::Result<()> {
windmill_common::instance_config::resolve_env_refs(&mut config.global_settings)
.map_err(|var| anyhow::anyhow!("environment variable '{var}' not found"))?;
let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish());
tracing::info!("Connecting to database...");
let db = crate::db_connect::initial_connection().await?;
config.sync_to_db(&db).await?;
init_declarative_writer_tracing(&db, "windmill-sync-config", early_logs).await;
let author = windmill_audit::instance_config::declarative_sync_author();
let synced = config
.sync_to_db(
&db,
&windmill_audit::instance_config::InstanceConfigAuditor {
db: &db,
author: &author,
source: Some(
windmill_audit::instance_config::DeclarativeSource::SyncConfig,
),
},
)
.await;
// Also on failure: the settings written before it are already audited, and the
// process exits before the exporter's next scheduled batch.
windmill_common::otel_oss::flush_otel_logs().await;
synced?;
tracing::info!("Synced instance config from {path}");
return Ok(());
}
#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))]
"oidc-signer" => {
tracing_subscriber::fmt::init();
windmill_api::oidc_signer_tools_ee::run_cli(std::env::args().skip(2).collect()).await?;
return Ok(());
}
#[cfg(feature = "operator")]
"operator" => {
tracing_subscriber::fmt::init();
let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish());
tracing::info!("Starting Windmill Kubernetes operator...");
tracing::info!("Connecting to database...");
@@ -842,6 +667,7 @@ async fn windmill_main() -> anyhow::Result<()> {
operator_killpill_rx,
)
.await?;
init_declarative_writer_tracing(&db, "windmill-operator", early_logs).await;
#[cfg(all(feature = "enterprise", feature = "private"))]
tokio::spawn(async move {
@@ -968,19 +794,7 @@ async fn windmill_main() -> anyhow::Result<()> {
(Connection::Sql(db), None, None)
};
let environment = if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") {
environment
} else {
load_base_url(&conn)
.await
.unwrap_or_else(|_| "local".to_string())
.trim_start_matches("https://")
.trim_start_matches("http://")
.split(".")
.next()
.unwrap_or_else(|| "local")
.to_string()
};
let environment = otel_environment(&conn).await;
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment);
@@ -1015,8 +829,10 @@ async fn windmill_main() -> anyhow::Result<()> {
.map(|v| v.to_lowercase() == "true" || v == "1")
.unwrap_or(false)
{
if let Err(e) = sync_cached_resource_types(db).await {
tracing::warn!("Failed to sync cached resource types: {:#}", e);
match hub_resource_types::sync_from_image_cache(db).await {
Ok(Some(outcome)) => tracing::info!("{}", outcome.summary()),
Ok(None) => {}
Err(e) => tracing::warn!("Failed to sync cached resource types: {:#}", e),
}
}
}
@@ -1234,6 +1050,15 @@ Windmill Community Edition {GIT_VERSION}
// empty script cache, refilled from the database.
windmill_common::cache::script::clear();
// Before the first load: it can sign a token for instance object storage.
#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))]
windmill_common::oidc_ee::init_oidc_signing(
&conn,
server_mode,
killpill_rx.resubscribe(),
)
.await?;
initial_load(
&conn,
killpill_tx.clone(),
@@ -1748,6 +1573,7 @@ Windmill Community Edition {GIT_VERSION}
if server_mode {
if let Some(db) = conn.as_sql() {
schedule_stats(&db, &HTTP_CLIENT).await;
hub_resource_types::spawn_daily_sync(db.clone(), killpill_tx.subscribe());
}
}
@@ -2322,6 +2148,12 @@ async fn process_notify_event(
RESTART_COORDINATION_SETTING => {
// Internal coordination key for staggered restarts, no action needed
}
SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING => {
// Read by the daily resource type sync each time it runs
}
"oidc_external_signer" => {
// Written by servers that sign with an external signer, read when signing
}
"plain_emails_telemetry" => {
let enabled = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'"
@@ -2407,6 +2239,8 @@ pub async fn run_workers(
&*PY311_CACHE_DIR,
&*PY312_CACHE_DIR,
&*PY313_CACHE_DIR,
&*PY314_CACHE_DIR,
&*PY315_CACHE_DIR,
&*BUN_BUNDLE_CACHE_DIR,
&*GO_CACHE_DIR,
&*GO_BIN_CACHE_DIR,
backend/src/monitor.rs
+60 -1
View File
@@ -427,6 +427,8 @@ pub async fn initial_load(
native_mode,
// an agent worker never reads its group's config, only its token
object_store_cache_config: None,
// nor whether it is paused: the server refuses it jobs instead
paused: false,
}));
}
}
@@ -4134,6 +4136,20 @@ pub async fn monitor_db(
}
_ => {}
}
match windmill_common::runnable_job_stats::cleanup_old_runnable_job_stats(
db,
windmill_common::runnable_job_stats::RETENTION_DAYS,
)
.await
{
Ok(count) if count > 0 => {
tracing::info!("Deleted {} old runnable job stats rows", count);
}
Err(e) => {
tracing::error!("Error cleaning up runnable job stats: {:?}", e);
}
_ => {}
}
}
}
};
@@ -4433,6 +4449,42 @@ pub async fn monitor_db(
}
};
// Drop the data table role set logins nobody has connected with for a month. Every ~40 min,
// spawned for the same reason as the credential maintenance above: it opens a connection per
// managed database for each login it drops.
let datatable_role_set_sweep_f = async {
if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) {
if let Some(db) = conn.as_sql() {
let db = db.clone();
tokio::spawn(async move {
if let Err(e) =
windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await
{
tracing::error!("Error sweeping unused data table role set logins: {e}");
}
});
}
}
};
// Re-encrypt the legacy-format secret variables once new values are written in AES-256-GCM.
// Every ~5 min until one pass has gone through every workspace; spawned so a large instance
// does not hold up the join below.
let legacy_secret_upgrade_f = async {
if server_mode
&& iteration.is_some()
&& iteration.as_ref().unwrap().should_run(30)
&& windmill_common::variables::legacy_secret_upgrade_pending()
{
if let Some(db) = conn.as_sql() {
let db = db.clone();
tokio::spawn(async move {
windmill_common::variables::upgrade_legacy_secret_variables(&db).await
});
}
}
};
// Delete the AI session backups older than their workspace's retention. Every ~40 min
// (240 iterations at the default 10 s, the most a u8 `should_run` counts): the retention
// counts in days. Spawned for the same reason as the credential maintenance above, a
@@ -4504,7 +4556,9 @@ pub async fn monitor_db(
cleanup_scheduled_job_deletions_f,
git_auto_pull_f,
git_credential_maintenance_f,
datatable_role_set_sweep_f,
ai_session_retention_f,
legacy_secret_upgrade_f,
pipeline_freshness_watchdog_f,
reconcile_unarmed_schedules_f,
stranded_jobs_f,
@@ -5689,8 +5743,13 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b
.as_ref()
.is_some_and(|dws| !dws.is_empty());
// Pausing or resuming must not restart the workers, dedicated ones included: a reload
// whose only difference is the flag skips every restart below.
let only_pause_changed = wc.paused != config.paused
&& WorkerConfig { paused: wc.paused, ..config.clone() } == **wc;
if **wc != config || has_dedicated {
if kill_if_change {
if kill_if_change && !only_pause_changed {
if has_dedicated
|| wc.dedicated_worker != config.dedicated_worker
|| wc.dedicated_workers != config.dedicated_workers
backend/summarized_schema.txt
+2 -1
View File
@@ -180,6 +180,7 @@ resource_type: workspace_id(char), name(char), schema(jsonb), description(text),
FK: (workspace_id) -> workspace(id)
resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool)
FK: (flow) -> v2_job_queue(id)
runnable_job_stats: hour(bigint), workspace_id(char), runnable_path(char), worker_group(text), job_count(int), total_duration_ms(bigint), max_memory_peak(int), sum_memory_peak(bigint), memory_sample_count(int), total_cpu_ms(bigint)
runnable_settings: hash(bigint), debouncing_settings(bigint), concurrency_settings(bigint)
schedule: workspace_id(char), path(char), edited_by(char), edited_at(ts), schedule(char), enabled(bool), script_path(char), args(jsonb), extra_perms(jsonb), is_flow(bool), email(char), error(text), timezone(char), on_failure(char), on_recovery(char), on_failure_times(int), on_failure_exact(bool), on_failure_extra_args(jsonb), on_recovery_times(int), on_recovery_extra_args(jsonb), ws_error_handler_muted(bool), retry(jsonb), summary(char), no_flow_overlap(bool), tag(char), paused_until(ts), on_success(char), on_success_extra_args(jsonb), cron_version(text), description(text), dynamic_skip(char), labels(text[])
FK: (workspace_id) -> workspace(id)
@@ -219,7 +220,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work
windmill_migrations: name(text), created_at(ts)
worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint)
FK: (workspace_id) -> workspace(id)
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[])
worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool)
workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool)
FK: (parent_workspace_id) -> workspace(id)
workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts)
backend/tests/ai_decision.rs
+1 -1
View File
@@ -117,7 +117,7 @@ async fn test_ai_decision_answers_route_a_following_branch(
"expr": "results.d.output.intent.choice === 'refund'",
"modules": [{"id": "r", "value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(choice){ return 'handled ' + choice }",
"input_transforms": {"choice": {
"type": "javascript",
backend/tests/app_ctx_inputs.rs
+88
View File
@@ -0,0 +1,88 @@
//! A context input a raw app runnable declares (`fields.<arg> = { type: "ctx", .. }`) is
//! filled by the server from the viewer's session. Whatever the request sends under that
//! name is discarded: a runnable filters rows on it, so a viewer who could set it would
//! read someone else's.
//!
//! Users from the `base` fixture:
//! test-user (admin, token SECRET_TOKEN)
//! test-user-2 (developer, token SECRET_TOKEN_2)
use serde_json::json;
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
const ADMIN_TOKEN: &str = "SECRET_TOKEN";
const VIEWER_TOKEN: &str = "SECRET_TOKEN_2";
const APP_PATH: &str = "u/test-user/ctx_app";
const CONTENT: &str = "export function main(who: string) { return who }";
// sha256 of CONTENT, the key an inline runnable is pinned under.
const CONTENT_SHA: &str = "6e198199621d57e4fed834510cb0c2692274aa4cbe976e1704fab2806be9d744";
fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder {
builder.header("Authorization", format!("Bearer {}", token))
}
#[sqlx::test(fixtures("base"))]
async fn a_declared_ctx_input_ignores_the_value_the_viewer_sends(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let ws = format!("http://localhost:{port}/api/w/test-workspace");
let client = reqwest::Client::new();
let resp = authed(client.post(format!("{ws}/apps/create")), ADMIN_TOKEN)
.json(&json!({
"path": APP_PATH,
"summary": "ctx inputs",
"value": {
"runnables": {
"whoami": {
"type": "inline",
"inlineScript": { "language": "bun", "content": CONTENT },
"fields": { "who": { "type": "ctx", "ctx": "username" } }
}
}
},
// The policy says nothing of context inputs: the server reads the
// declaration from the deployed runnable.
"policy": {
"execution_mode": "anonymous",
"triggerables_v2": {
format!("whoami:rawscript/{CONTENT_SHA}"): {
"static_inputs": {},
"one_of_inputs": {}
}
}
}
}))
.send()
.await?;
assert!(resp.status().is_success(), "{}", resp.text().await?);
// Both spellings a viewer could send: a forged literal, and no argument at all.
for args in [json!({ "who": "test-user" }), json!({})] {
let resp = authed(
client.post(format!("{ws}/apps_u/execute_component/{APP_PATH}")),
VIEWER_TOKEN,
)
.json(&json!({
"component": "whoami",
"args": args,
"raw_code": { "language": "bun", "content": CONTENT }
}))
.send()
.await?;
assert!(resp.status().is_success(), "{}", resp.text().await?);
let job_id: uuid::Uuid = resp.text().await?.parse()?;
let queued: serde_json::Value = sqlx::query_scalar("SELECT args FROM v2_job WHERE id = $1")
.bind(job_id)
.fetch_one(&db)
.await?;
assert_eq!(queued["who"], json!("test-user-2"), "sent {args}");
}
Ok(())
}
backend/tests/debounce_e2e.rs
+1 -1
View File
@@ -33,7 +33,7 @@ mod debounce_e2e {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export async function main(items: any[]) { return items }",
"input_transforms": {
"items": { "type": "javascript", "expr": "flow_input.items" }
backend/tests/drafts_delete_for_user.rs
+85
View File
@@ -0,0 +1,85 @@
//! Only a workspace admin can delete a draft that belongs to another user.
//!
//! A discard through `drafts/update` is scoped to the caller's own row, so it
//! never reaches someone else's draft. `drafts/delete_for_user` is the admin
//! route for that, and it must stay closed to everyone else.
use serde_json::{json, Value};
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
#[sqlx::test(fixtures("base"))]
async fn test_only_admin_deletes_another_users_draft(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/drafts");
let client = reqwest::Client::new();
let path = "script/u/test-user-2/theirs";
let save = |token: &'static str| {
client
.post(format!("{base}/update/{path}"))
.header("Authorization", format!("Bearer {token}"))
.json(&json!({ "value": { "path": "u/test-user-2/theirs", "content": "" } }))
.send()
};
let resp = save("SECRET_TOKEN_2").await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
let delete = |token: &'static str| {
client
.post(format!(
"{base}/delete_for_user/{path}?username=test-user-2"
))
.header("Authorization", format!("Bearer {token}"))
.send()
};
let own_draft = || async {
client
.get(format!("{base}/get_own/{path}"))
.header("Authorization", "Bearer SECRET_TOKEN_2")
.send()
.await?
.json::<Value>()
.await
};
let resp = delete("SECRET_TOKEN_3").await?;
assert_eq!(resp.status(), 403, "{}", resp.text().await?);
assert!(
!own_draft().await?.is_null(),
"a non-admin deleted the draft"
);
let resp = delete("SECRET_TOKEN").await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
assert!(own_draft().await?.is_null(), "the draft is still there");
let resp = delete("SECRET_TOKEN").await?;
assert_eq!(resp.status(), 404, "{}", resp.text().await?);
// Without a username the delete takes every draft at the path.
for token in ["SECRET_TOKEN_2", "SECRET_TOKEN"] {
let resp = save(token).await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
}
let delete_all = |token: &'static str| {
client
.post(format!("{base}/delete_for_user/{path}"))
.header("Authorization", format!("Bearer {token}"))
.send()
};
let resp = delete_all("SECRET_TOKEN_3").await?;
assert_eq!(resp.status(), 403, "{}", resp.text().await?);
let resp = delete_all("SECRET_TOKEN").await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
let left: i64 = sqlx::query_scalar(
"SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user-2/theirs'",
)
.fetch_one(&db)
.await?;
assert_eq!(left, 0, "drafts survived a path-wide delete");
Ok(())
}
backend/tests/end_user_email.rs
+116 -82
View File
@@ -1,10 +1,8 @@
//! Tests for WM_END_USER_EMAIL environment variable.
//!
//! These tests verify that WM_END_USER_EMAIL is populated with the authenticated
//! user's email when executing app components.
//!
//! TODO: Add tests for scripts and flows once public execution endpoints are identified.
//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup.
//! user's email when executing app components and when running scripts and flows directly.
//! Only apps support non-workspace-member execution via OptAuthed + token lookup.
use serde_json::json;
use sqlx::{Pool, Postgres};
@@ -27,40 +25,6 @@ fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuil
builder.header("Authorization", format!("Bearer {}", token))
}
// TODO: Script tests - need to identify public execution endpoints for non-workspace-members
// async fn run_script(port: u16, token: &str) -> anyhow::Result<String> {
// let url = format!(
// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email",
// port
// );
// let resp = authed(client().post(&url), token)
// .json(&json!({}))
// .send()
// .await?;
// if !resp.status().is_success() {
// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?);
// }
// Ok(resp.json::<serde_json::Value>().await?
// .as_str().unwrap_or("").to_string())
// }
// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members
// async fn run_flow(port: u16, token: &str) -> anyhow::Result<String> {
// let url = format!(
// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow",
// port
// );
// let resp = authed(client().post(&url), token)
// .json(&json!({}))
// .send()
// .await?;
// if !resp.status().is_success() {
// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?);
// }
// Ok(resp.json::<serde_json::Value>().await?
// .as_str().unwrap_or("").to_string())
// }
/// Create an app with inline script via API
async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> {
let url = format!("http://localhost:{}/api/w/test-workspace/apps/create", port);
@@ -74,8 +38,8 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<
"subgrids": {},
"hiddenInlineScripts": [{
"name": "get_email",
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }",
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }",
"path": "f/test/email_app/get_email"
}]
},
@@ -89,7 +53,7 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<
"one_of_inputs": {}
},
// SHA256 hash of raw_code content for anonymous execution
"rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": {
"rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": {
"static_inputs": {},
"one_of_inputs": {}
}
@@ -120,8 +84,8 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res
"css": "",
"inlineScripts": [{
"name": "get_email",
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }"
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }"
}]
},
"policy": {
@@ -134,7 +98,7 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res
"one_of_inputs": {}
},
// SHA256 hash of raw_code content for anonymous execution
"rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": {
"rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": {
"static_inputs": {},
"one_of_inputs": {}
}
@@ -233,8 +197,8 @@ async fn run_app_inline_script(
"args": {},
"component": "get_email",
"raw_code": {
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }",
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }",
"path": format!("{}/get_email", app_path)
}
});
@@ -270,8 +234,8 @@ async fn run_raw_app_inline_script(
"args": {},
"component": "get_email",
"raw_code": {
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }"
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }"
}
});
if force_viewer {
@@ -312,41 +276,111 @@ async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Re
anyhow::bail!("timeout waiting for job result")
}
// TODO: Script tests - need to identify public execution endpoints for non-workspace-members
// #[cfg(feature = "deno_core")]
// #[sqlx::test(fixtures("base", "end_user_email"))]
// async fn test_script_wm_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
// initialize_tracing().await;
// set_jwt_secret().await;
// let server = ApiServer::start(db.clone()).await?;
// let port = server.addr.port();
//
// in_test_worker(Connection::Sql(db.clone()), async move {
// let result = run_script(port, SAME_WS_TOKEN).await?;
// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email");
// Ok::<(), anyhow::Error>(())
// }, port).await?;
//
// Ok(())
// }
async fn run_direct(port: u16, token: &str, route: &str) -> anyhow::Result<String> {
let url = format!(
"http://localhost:{}/api/w/test-workspace/jobs/run/{}",
port, route
);
let resp = authed(client().post(&url), token)
.json(&json!({}))
.send()
.await?;
if !resp.status().is_success() {
anyhow::bail!(
"run {route} failed: {} - {}",
resp.status(),
resp.text().await?
);
}
let job_id = resp.text().await?;
wait_for_job_result(port, token, &job_id).await
}
// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members
// #[cfg(feature = "deno_core")]
// #[sqlx::test(fixtures("base", "end_user_email"))]
// async fn test_flow_wm_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
// initialize_tracing().await;
// set_jwt_secret().await;
// let server = ApiServer::start(db.clone()).await?;
// let port = server.addr.port();
//
// in_test_worker(Connection::Sql(db.clone()), async move {
// let result = run_flow(port, SAME_WS_TOKEN).await?;
// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email");
// Ok::<(), anyhow::Error>(())
// }, port).await?;
//
// Ok(())
// }
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base", "end_user_email"))]
async fn test_direct_run_wm_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
set_jwt_secret().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
in_test_worker(
Connection::Sql(db.clone()),
async move {
let result = run_direct(port, SAME_WS_TOKEN, "p/f/test/get_end_user_email").await?;
assert_eq!(result, SAME_WS_EMAIL, "a script run should see its caller");
let result =
run_direct(port, SAME_WS_TOKEN, "f/f/test/get_end_user_email_flow").await?;
assert_eq!(
result, SAME_WS_EMAIL,
"every flow step should see the flow's caller"
);
Ok::<(), anyhow::Error>(())
},
port,
)
.await?;
Ok(())
}
/// A job's `WM_TOKEN` names the identity the job runs as, which differs from its end user when
/// the job runs on behalf of another account. A run it starts must keep the job's end user.
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base", "end_user_email"))]
async fn test_job_token_run_keeps_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
set_jwt_secret().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let calling_job = uuid::Uuid::new_v4();
sqlx::query(
"INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email)
VALUES ($1, 'test2@windmill.dev', 'test-user-2', false, false, '{}', '{}', 'test-workspace', $2)",
)
.bind(calling_job)
.bind(SAME_WS_EMAIL)
.execute(&db)
.await?;
let wm_token = windmill_common::auth::create_jwt_token(
windmill_common::db::Authed {
email: "test2@windmill.dev".to_string(),
username: "test-user-2".to_string(),
is_admin: false,
is_operator: false,
groups: vec!["all".to_string()],
folders: vec![],
scopes: None,
token_prefix: None,
},
"test-workspace",
3600,
Some(calling_job),
None,
None,
None,
)
.await?;
in_test_worker(
Connection::Sql(db.clone()),
async move {
let result = run_direct(port, &wm_token, "p/f/test/get_end_user_email").await?;
assert_eq!(
result, SAME_WS_EMAIL,
"the calling job's end user, not its run-as identity"
);
Ok::<(), anyhow::Error>(())
},
port,
)
.await?;
Ok(())
}
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base", "end_user_email"))]

Some files were not shown because too many files have changed in this diff Show More