starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-13 00:03:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into hugo/git-1044-bug-mcp-resource-var-in-url-is-not-resolved-by-the-ai-agent

Browse Source
This commit is contained in:
hugocasa
2026-10-12 09:45:37 +02:00
parent ded49c51f8 9fa9b77926
commit aba65c8d23
456 changed files with 12738 additions and 4967 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/DockerfileBackendTests
+1 -1
View File
@@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
# UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv
ENV TZ=Etc/UTC
.github/workflows/backend-test-windows.yml
+2 -2
View File
@@ -63,7 +63,7 @@ jobs:
- uses: denoland/setup-deno@v2
with:
# Pin to the Deno version shipped in the runtime image (Dockerfile) so CI
# Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI
# tests what production runs, instead of floating on the latest v2.x.
deno-version: 2.2.1
@@ -81,7 +81,7 @@ jobs:
- uses: astral-sh/setup-uv@v7
with:
version: "0.11.24"
version: "0.13.0"
- uses: shivammathur/setup-php@v2
with:
.github/workflows/backend-test.yml
+2 -2
View File
@@ -50,7 +50,7 @@ jobs:
dotnet-version: "9.0.x"
- uses: denoland/setup-deno@v2
with:
# Pin to the Deno version shipped in the runtime image (Dockerfile) so CI
# Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI
# tests what production runs, instead of floating on the latest v2.x.
deno-version: 2.2.1
- uses: actions/setup-go@v7
@@ -64,7 +64,7 @@ jobs:
node-version: "24"
- uses: astral-sh/setup-uv@v7
with:
version: "0.11.24"
version: "0.13.0"
- uses: shivammathur/setup-php@v2
with:
php-version: "8.3"
.github/workflows/benchmark.yml
+14 -14
View File
@@ -27,7 +27,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
@@ -111,7 +111,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
@@ -125,7 +125,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -136,7 +136,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -147,7 +147,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -188,7 +188,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
@@ -202,7 +202,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -213,7 +213,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -224,7 +224,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -235,7 +235,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -246,7 +246,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -257,7 +257,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
@@ -268,7 +268,7 @@ jobs:
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
MODE: worker
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always
steps:
@@ -309,7 +309,7 @@ jobs:
DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
WORKER_GROUP: main
WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets
options: >-
--pull always --health-interval 10s --health-timeout 5s
--health-retries 5 --health-cmd "curl
.release-please-manifest.json
+1 -1
View File
@@ -1,3 +1,3 @@
{
".": "1.828.0"
".": "1.830.0"
}
AGENTS.md
+3 -2
View File
@@ -37,8 +37,9 @@ Open-source platform for internal tools, workflows, API integrations, background
- **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are
enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option`
fields and a jsonb merge rather than a serde default
- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace setting that lets
operators compose flows, what the composition check must cover, and why it costs a full seat
- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace settings that let
operators compose flows and full-code apps, what the composition check must cover, which
raw-app endpoint stays closed on purpose, and why either right costs a full seat
- **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation
scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and
that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates
CHANGELOG.md
+50
View File
@@ -1,5 +1,55 @@
# Changelog
## [1.830.0](https://github.com/windmill-labs/windmill/compare/v1.829.0...v1.830.0) (2026-10-09)
### Features
* add a jobs:run:dependencies token scope for lock generation ([#11643](https://github.com/windmill-labs/windmill/issues/11643)) ([1dc4ed5](https://github.com/windmill-labs/windmill/commit/1dc4ed5e7fb86c9658f1f877e7b92cd9dab5e283))
* let operators compose full-code apps when the workspace grants it ([#11229](https://github.com/windmill-labs/windmill/issues/11229)) ([02e4697](https://github.com/windmill-labs/windmill/commit/02e4697157112581e9e2a9303180077126db714b))
* operators get AI sessions ([#11443](https://github.com/windmill-labs/windmill/issues/11443)) ([338c4e9](https://github.com/windmill-labs/windmill/commit/338c4e921dc3185d886c8b040f253285adf20efb))
* replace the hub_sync script with an opt-in daily server sync ([#11624](https://github.com/windmill-labs/windmill/issues/11624)) ([2b55a8a](https://github.com/windmill-labs/windmill/commit/2b55a8afbc8a86c6cebe97b1cab02c313b986f22))
* retry transient AI provider failures before output reaches readers ([#11623](https://github.com/windmill-labs/windmill/issues/11623)) ([d3ddef0](https://github.com/windmill-labs/windmill/commit/d3ddef0b7ec4705aa578c4eca24b96e9152485bd))
* stop offering deno for new scripts, ship it only on full images ([#11648](https://github.com/windmill-labs/windmill/issues/11648)) ([cda7883](https://github.com/windmill-labs/windmill/commit/cda78830fde1b96bd3de2674e23c245b50e3c051))
### Bug Fixes
* **cli:** count superadmins as able to preserve ownership on push ([#11653](https://github.com/windmill-labs/windmill/issues/11653)) ([70db567](https://github.com/windmill-labs/windmill/commit/70db567a78b1e1bc9e2dcc9ff2139951d778117e))
* harden app secret validation ([#11655](https://github.com/windmill-labs/windmill/issues/11655)) ([8556a7b](https://github.com/windmill-labs/windmill/commit/8556a7bebe093e0bd61c33d67415fd553f0070ad))
* open the chat + menu without waiting on the skills and mcp fetches ([#11645](https://github.com/windmill-labs/windmill/issues/11645)) ([1fea1aa](https://github.com/windmill-labs/windmill/commit/1fea1aa6609d644e02bac8d16186f9a2dfcb63cd))
* say Running on a chat run card only once its job exists ([#11627](https://github.com/windmill-labs/windmill/issues/11627)) ([d046187](https://github.com/windmill-labs/windmill/commit/d046187e3edd01535240c6080b483afe154ae189))
## [1.829.0](https://github.com/windmill-labs/windmill/compare/v1.828.0...v1.829.0) (2026-10-09)
### Features
* add an api base url instance setting for displayed urls ([#11628](https://github.com/windmill-labs/windmill/issues/11628)) ([6acf185](https://github.com/windmill-labs/windmill/commit/6acf1858540e83b65b5218c50c98af692092d22e))
* add Pi template to the AI sandbox picker ([#11619](https://github.com/windmill-labs/windmill/issues/11619)) ([a85d0b7](https://github.com/windmill-labs/windmill/commit/a85d0b76cc494aaa4a484a273ec534bdc2958102))
* agent nits: multi-select, memory-off notice, agent step label ([#11622](https://github.com/windmill-labs/windmill/issues/11622)) ([c52ba2f](https://github.com/windmill-labs/windmill/commit/c52ba2fd30e19986522420ff02fae224a075b4f6))
* export worker cpu, memory and occupancy metrics ([#11629](https://github.com/windmill-labs/windmill/issues/11629)) ([5d90f96](https://github.com/windmill-labs/windmill/commit/5d90f96863b64e9bc88ed35429f8068a6a788d1e))
* let a job see that its worker is draining ([#11612](https://github.com/windmill-labs/windmill/issues/11612)) ([588d846](https://github.com/windmill-labs/windmill/commit/588d8463dfa1c8936b8753b5f0fd794ecb115203))
* let workspace admins delete other users' drafts ([#11644](https://github.com/windmill-labs/windmill/issues/11644)) ([68376fc](https://github.com/windmill-labs/windmill/commit/68376fc011ef6406a820c69217058316514d3f8d))
* per-script resource rollup and heaviest scripts table ([#11631](https://github.com/windmill-labs/windmill/issues/11631)) ([442357d](https://github.com/windmill-labs/windmill/commit/442357dbe8db0c52995572acd7385dedf4efdc42))
* pipeline editor: assets-only view, Fix actions and draft triggers ([#11440](https://github.com/windmill-labs/windmill/issues/11440)) ([3dae553](https://github.com/windmill-labs/windmill/commit/3dae553576ba23b2c1f56e4deac01305c4645c01))
### Bug Fixes
* advertise the actual signing algorithm in OIDC discovery ([#11637](https://github.com/windmill-labs/windmill/issues/11637)) ([98fc65a](https://github.com/windmill-labs/windmill/commit/98fc65a16bbe2bc9515cdd5443304053c4322274))
* ask a workspace for its session backups before pushing to it ([#11641](https://github.com/windmill-labs/windmill/issues/11641)) ([2ea846f](https://github.com/windmill-labs/windmill/commit/2ea846f563f5a77cbd3eae2eddee9e38a3700d00))
* keep instance settings users sub-tabs from being squashed ([#11633](https://github.com/windmill-labs/windmill/issues/11633)) ([fb22e5c](https://github.com/windmill-labs/windmill/commit/fb22e5ce368eba9146e0d51b8fdabb52ca9f68f0))
* keep raw app deploy panel settings shared with the editor ([#11616](https://github.com/windmill-labs/windmill/issues/11616)) ([0a95ea4](https://github.com/windmill-labs/windmill/commit/0a95ea47facaede2545cc4741d4a70c1c847e11a))
* load schedules once per page load, poll critical alerts every 30s ([#11639](https://github.com/windmill-labs/windmill/issues/11639)) ([1b4e686](https://github.com/windmill-labs/windmill/commit/1b4e686b2f5de3138df0c3b6ed627ea0fe0a2f7d))
* name the open workspace settings tab in the page header breadcrumb ([#11617](https://github.com/windmill-labs/windmill/issues/11617)) ([e233908](https://github.com/windmill-labs/windmill/commit/e233908c1e99a246dc7c94fa86ea82c5cbe9d6b3))
* **python:** add missing import-to-package mappings for common modules ([#11638](https://github.com/windmill-labs/windmill/issues/11638)) ([ab8606f](https://github.com/windmill-labs/windmill/commit/ab8606f37921913ef1410da735abb5742c025225))
* send hub script completions to the workspace handlers ([#11613](https://github.com/windmill-labs/windmill/issues/11613)) ([d685332](https://github.com/windmill-labs/windmill/commit/d685332771dcdfc718d7a2077fe1cdb3b323909b))
* set WM_END_USER_EMAIL on direct script and flow runs ([#11636](https://github.com/windmill-labs/windmill/issues/11636)) ([2bdeceb](https://github.com/windmill-labs/windmill/commit/2bdeceba7fb6815cf3150ecb856afb8eb3872e44))
* store a WAC parent's first start and wall duration ([#11625](https://github.com/windmill-labs/windmill/issues/11625)) ([8f28a58](https://github.com/windmill-labs/windmill/commit/8f28a5860595dbe558e6777a06bb97b6ec6a3e67))
* sync back volume files rewritten to the same size ([#11620](https://github.com/windmill-labs/windmill/issues/11620)) ([2758920](https://github.com/windmill-labs/windmill/commit/2758920f1182ddd12e6473a644a4a966a435522e))
* uniform decrypt error and clear stored credentials on logout ([#11632](https://github.com/windmill-labs/windmill/issues/11632)) ([987c321](https://github.com/windmill-labs/windmill/commit/987c321ebcd305df79fa945e941abc2b2e1d3e95))
## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08)
Dockerfile
+1 -4
View File
@@ -83,7 +83,6 @@ COPY /python-client/docs/ /frontend/static/pydocs/
RUN npm run generate-backend-client
ENV NODE_OPTIONS "--max-old-space-size=8192"
ARG VITE_BASE_URL ""
# Must be declared for the build-arg to reach the bundle. See frontend/svelte.config.js.
ARG WM_BUILD_VERSION=""
# Read more about macro in docker/dev.nu
@@ -242,7 +241,7 @@ ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv
# Preinstall python runtimes to temp build location (will copy with world-writable perms later)
# --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run
@@ -287,8 +286,6 @@ COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so ${APP}/libwindmill_duckdb_ffi_internal.so
COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun
# Install windmill CLI
README.md
+2 -2
View File
@@ -150,7 +150,7 @@ You can run scripts locally by passing the right environment variables for the `
- **Frontend**: Svelte 5
- **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation
- **Runtimes**:
- TypeScript/JavaScript: Bun (default) and Deno
- TypeScript/JavaScript: Bun (default); Deno only on the `windmill-full` images
- Python: python3 with uv for dependency management
- Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible
@@ -318,7 +318,7 @@ running options.
This will also avoid compile time issue with sqlx's `query!` macro.
2. (optional, linux only) Install [nsjail](https://github.com/google/nsjail) and have it accessible in
your PATH
3. Install bun, deno and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun`,`/usr/bin/deno`, and
3. Install bun and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun` and
`/usr/local/bin/python3` or set the corresponding environment variables.
4. (optional) Install the [lld linker](https://lld.llvm.org/)
5. Go to `frontend/`:
backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json → backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true)\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>', true)\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde"
"hash": "009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6"
}
backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json → backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730"
"hash": "1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e"
}
backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217"
}
backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM runnable_job_stats s WHERE s.hour >= $1 AND NOT EXISTS (SELECT 1 FROM workspace w WHERE w.id = s.workspace_id)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8"
}
backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json
Generated
+40
View File
@@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",\n COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\",\n COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",\n COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"\n FROM workspace_settings WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "flows!",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "apps!",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "schedules!",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "triggers!",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
null,
null,
null
]
},
"hash": "1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80"
}
backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json
Generated
-20
View File
@@ -1,20 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Jsonb",
"Text",
"Varchar",
"Bool",
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead"
}
backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO runnable_job_stats\n (hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n SELECT s.hour, s.workspace_id, s.runnable_path, s.worker_group, s.job_count,\n s.total_duration_ms, s.max_memory_peak, s.sum_memory_peak,\n s.memory_sample_count, s.total_cpu_ms\n FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::int[], $6::bigint[],\n $7::int[], $8::bigint[], $9::int[], $10::bigint[])\n AS s(hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n JOIN workspace w ON w.id = s.workspace_id\n ORDER BY s.hour, s.workspace_id, s.runnable_path, s.worker_group\n ON CONFLICT (hour, workspace_id, runnable_path, worker_group) DO UPDATE SET\n job_count = runnable_job_stats.job_count + EXCLUDED.job_count,\n total_duration_ms = runnable_job_stats.total_duration_ms + EXCLUDED.total_duration_ms,\n max_memory_peak = GREATEST(runnable_job_stats.max_memory_peak, EXCLUDED.max_memory_peak),\n sum_memory_peak = runnable_job_stats.sum_memory_peak + EXCLUDED.sum_memory_peak,\n memory_sample_count = runnable_job_stats.memory_sample_count + EXCLUDED.memory_sample_count,\n total_cpu_ms = runnable_job_stats.total_cpu_ms + EXCLUDED.total_cpu_ms\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8Array",
"TextArray",
"TextArray",
"TextArray",
"Int4Array",
"Int8Array",
"Int4Array",
"Int8Array",
"Int4Array",
"Int8Array"
]
},
"nullable": []
},
"hash": "24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875"
}
backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json → backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -8,5 +8,5 @@
},
"nullable": []
},
"hash": "7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877"
"hash": "344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d"
}
backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Jsonb",
"Text",
"Text",
"Bool",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34"
}
backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json → backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95"
"hash": "42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779"
}
backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json → backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4"
"hash": "69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927"
}
backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json
Generated
+45
View File
@@ -0,0 +1,45 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e"
}
backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json → backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -10,5 +10,5 @@
},
"nullable": []
},
"hash": "51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e"
"hash": "8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0"
}
backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json → backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2",
"query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2",
"describe": {
"columns": [
{
@@ -31,5 +31,5 @@
null
]
},
"hash": "7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec"
"hash": "8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3"
}
backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json
Generated
-31
View File
@@ -1,31 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
false
]
},
"hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d"
}
backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json
Generated
-33
View File
@@ -1,33 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "duration_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray",
"Uuid",
"Text"
]
},
"nullable": [
false
]
},
"hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5"
}
backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM runnable_job_stats WHERE hour < $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e"
}
backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d"
}
backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665"
}
backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json → backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c"
"hash": "b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a"
}
backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
null
]
},
"hash": "b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37"
}
backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json
Generated
+74
View File
@@ -0,0 +1,74 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT workspace_id, runnable_path, worker_group,\n SUM(job_count)::bigint AS \"job_count!\",\n SUM(total_duration_ms)::bigint AS \"total_duration_ms!\",\n MAX(max_memory_peak) AS \"max_memory_peak!\",\n SUM(sum_memory_peak)::bigint AS \"sum_memory_peak!\",\n SUM(memory_sample_count)::bigint AS \"memory_sample_count!\",\n SUM(total_cpu_ms)::bigint AS \"total_cpu_ms!\"\n FROM runnable_job_stats\n WHERE hour >= $1\n AND ($2::text IS NULL OR worker_group = $2)\n AND ($3::text IS NULL OR workspace_id = $3)\n GROUP BY workspace_id, runnable_path, worker_group\n ORDER BY CASE $4::text\n WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric\n WHEN 'cpu' THEN SUM(total_cpu_ms)\n ELSE SUM(total_duration_ms)\n END DESC, workspace_id, runnable_path, worker_group\n LIMIT $5\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "worker_group",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "job_count!",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "total_duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "max_memory_peak!",
"type_info": "Int4"
},
{
"ordinal": 6,
"name": "sum_memory_peak!",
"type_info": "Int8"
},
{
"ordinal": 7,
"name": "memory_sample_count!",
"type_info": "Int8"
},
{
"ordinal": 8,
"name": "total_cpu_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Text",
"Text",
"Text",
"Int8"
]
},
"nullable": [
false,
false,
false,
null,
null,
null,
null,
null,
null
]
},
"hash": "c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477"
}
backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json → backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ",
"query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n<empty>')\n ",
"describe": {
"columns": [],
"parameters": {
@@ -8,5 +8,5 @@
},
"nullable": []
},
"hash": "cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20"
"hash": "cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74"
}
backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json
Generated
+43
View File
@@ -0,0 +1,43 @@
{
"db_name": "PostgreSQL",
"query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "started_at?",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "duration_ms!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "last_run_ms!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid",
"Bool",
"Text",
"Bool",
"Varchar",
"Text",
"Bool",
"Int4",
"Int8",
"TextArray"
]
},
"nullable": [
true,
false,
null
]
},
"hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8"
}
backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM runnable_job_stats WHERE workspace_id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8"
}
backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int8",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd"
}
backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "executions",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323"
}
backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json → backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT COALESCE((ws.operator_settings->>'builder_flows')::boolean, false)) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
"query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT (COALESCE((ws.operator_settings->>'builder_flows')::boolean, false) OR COALESCE((ws.operator_settings->>'builder_apps')::boolean, false))) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg",
"describe": {
"columns": [
{
@@ -34,5 +34,5 @@
null
]
},
"hash": "cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b"
"hash": "efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b"
}
backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text",
"Int4",
"Float8"
]
},
"nullable": []
},
"hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed"
}
backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json
Generated
+59
View File
@@ -0,0 +1,59 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3\n AND ($4::text IS NULL OR email = $4)\n RETURNING 1 as \"one!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "one!",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "draft_kind",
"kind": {
"Enum": [
"script",
"flow",
"app",
"raw_app",
"resource",
"variable",
"trigger_schedule",
"trigger_webhook",
"trigger_default_email",
"trigger_email",
"trigger_http",
"trigger_websocket",
"trigger_postgres",
"trigger_kafka",
"trigger_nats",
"trigger_mqtt",
"trigger_sqs",
"trigger_gcp",
"trigger_azure",
"trigger_poll",
"trigger_cli",
"trigger_nextcloud",
"trigger_google",
"trigger_github",
"data_pipeline",
"trigger_amqp"
]
}
}
},
"Text"
]
},
"nullable": [
null
]
},
"hash": "f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18"
}
backend/AGENTS.md
+1 -1
View File
@@ -214,7 +214,7 @@ Whole classes of defect compile and unit-test clean:
- **Stack overflow from a large buffer in an async block.** An array declared across an
`.await` is baked into the future's state; once that future is boxed a few layers deep
by the job poller, two 16 KB arrays abort the worker *process* (`thread
'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers
'tokio-rt-worker' has overflowed its stack`). Heap-allocate read buffers
(`vec![0u8; N]`, not `[0u8; N]`).
- Deadlocks from draining only one of a child's pipes, missed cancellation or timeout
propagation, and anything depending on the real engine's output format.
backend/Cargo.lock
Generated
+1520 -840
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+31 -33
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.828.0"
version = "1.830.0"
authors.workspace = true
edition.workspace = true
@@ -88,7 +88,7 @@ members = [
exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"]
[workspace.package]
version = "1.828.0"
version = "1.830.0"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -451,7 +451,7 @@ headers = "^0"
hyper = { version = "^1", features = ["full"] }
hyper-tls = "^0.6"
hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] }
tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] }
tokio = { version = "=1.53.2", features = ["full", "tracing", "time"] }
tokio-stream = { version = "0.1.17" }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] }
@@ -488,9 +488,11 @@ arc-swap = "1"
ed25519-dalek = { version = "2", features = ["rand_core"] }
magic-crypt = "^3"
git-version = "^0"
malachite = "=0.4.18"
malachite-bigint = "=0.2.0"
rustpython-parser = "^0"
# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly,
# and to the same release.
ruff_python_parser = "=0.0.17"
ruff_python_ast = "=0.0.17"
ruff_text_size = "=0.0.17"
pep440_rs = "0.7.3"
php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" }
cron = "^0"
@@ -519,33 +521,29 @@ aws-sdk-rds = "^1"
async-trait = "0.1.88"
v8 = "=137.1.0" # Exact version NOTE: Do not forget to update version and hash in flake.nix
# deno_* pin set: deno v2.4.0 base, with deno_ast force-overridden to =0.51.0.
# Rationale: deno_ast 0.51.0 is the first version pulling swc_common =14.0.4,
# the first swc_common patch that dropped `pub use serde::__private as serde;`
# (the line that capped our workspace serde pin at =1.0.220). v2.4.0's other
# pins keep deno_tls at 0.196.0 which uses permissive `rustls ^0.23.11`,
# compatible with aws-sdk-bedrockruntime's `^0.23.31` requirement. deno_tls
# 0.198+ tightened that to exact `=0.23.28`, which would have made any
# meaningful deno bump resolver-impossible against aws-sdk.
deno_fetch = "0.233.0"
deno_tls = "0.196.0"
deno_console = "0.209.0"
deno_url = "0.209.0"
deno_webidl = "0.209.0"
deno_web = "0.240.0"
# Sibling release of the pinned deno_core 0.352 / deno_web 0.240 stack
# (its deps are deno_core ^0.352, deno_web ^0.240, deno_error =0.6.1). A newer
# deno_crypto would force bumping the whole deno stack.
deno_crypto = "0.223.0"
deno_io = "0.119.0"
deno_fs = "0.119.0"
deno_net = "0.201.0"
deno_core = "0.352.0"
# deno_* pin set: the crates deno published together on 2026-09-16 (deno_core
# 0.412).
# V8 is pinned exactly because flake.nix fetches the prebuilt archive of that
# version: update its version and hash there with any bump. No default features,
# so the pin leaves the feature set deno_core picks untouched.
v8 = { version = "=150.4.0", default-features = false }
# These release as a matched set: each pins its siblings, deno_error exactly,
# and deno_tls pins rustls exactly (=0.23.40, the workspace `rustls` pin below
# must equal it). A bump moves all of them at once.
# deno_ast is independent of that set and stays at 0.51.0: a newer one pulls
# swc_common 17, which would force bumping every swc_* pin the parsers use.
deno_fetch = "0.283.0"
deno_tls = "0.246.0"
deno_webidl = "0.259.0"
deno_web = "0.290.0"
deno_crypto = "0.273.0"
deno_net = "0.251.0"
deno_core = "0.412.0"
deno_ast = { version = "=0.51.0", features = ["transpiling"] }
deno_permissions = "0.68.0"
deno_telemetry = "0.31.0"
deno_error = "=0.6.1"
deno_permissions = "0.118.0"
deno_telemetry = "0.81.0"
deno_error = "=0.7.1"
sys_traits = { version = "0.1.28", features = ["real", "libc", "winapi"] }
rustls-pemfile = "2.2.0"
# only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed
@@ -682,7 +680,7 @@ aws-sdk-sqs = "^1.89.0"
aws-sdk-sts = "^1.91.0"
aws-sdk-sso = "=1.77.0"
aws-sdk-ssooidc = "=1.78.0"
rustls = "=0.23.35"
rustls = "=0.23.40"
async-once-cell = "0.5.4"
size = "0.5.0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
127575148c84bb6132d2946c0a95c6dea0f58bb2
06610ecc950ed64a2709d7f1aa8c7b49a95fdbae
backend/migrations/20261008165042_runnable_job_stats.down.sql
+1
View File
@@ -0,0 +1 @@
DROP TABLE IF EXISTS runnable_job_stats;
backend/migrations/20261008165042_runnable_job_stats.up.sql
+17
View File
@@ -0,0 +1,17 @@
-- Hourly resource rollup per runnable and worker group, fed by the workers.
-- Memory figures are in kB, like v2_job_completed.memory_peak.
-- No foreign key to workspace: every worker upserts here, and the key would cost each row
-- a lookup and a lock on its workspace. Workspace deletion removes the rows itself.
CREATE TABLE IF NOT EXISTS runnable_job_stats (
hour BIGINT NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
runnable_path VARCHAR(255) NOT NULL,
worker_group TEXT NOT NULL,
job_count INTEGER NOT NULL DEFAULT 0,
total_duration_ms BIGINT NOT NULL DEFAULT 0,
max_memory_peak INTEGER NOT NULL DEFAULT 0,
sum_memory_peak BIGINT NOT NULL DEFAULT 0,
memory_sample_count INTEGER NOT NULL DEFAULT 0,
total_cpu_ms BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (hour, workspace_id, runnable_path, worker_group)
);
backend/parsers/windmill-parser-py-asset/Cargo.toml
+2 -2
View File
@@ -11,6 +11,6 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql-asset.workspace = true
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
anyhow.workspace = true
backend/parsers/windmill-parser-py-asset/src/lib.rs
+168 -96
View File
@@ -1,5 +1,8 @@
use rustpython_ast::{Constant, Expr, ExprConstant, Visitor};
use rustpython_parser::{ast::Suite, Parse};
use ruff_python_ast::{
visitor::{walk_expr, walk_stmt, Visitor},
Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign,
};
use ruff_python_parser::parse_module;
use std::collections::HashMap;
use windmill_parser::asset_parser::{
asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind,
@@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{
use AssetUsageAccessType::*;
pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
let ast = Suite::parse(input, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(input)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() };
ast.into_iter()
.for_each(|stmt| assets_finder.visit_stmt(stmt));
assets_finder.visit_body(&ast);
for (kind, path, _) in assets_finder.var_identifiers.into_values() {
// if a db = wmill.datatable() was never used (e.g db.query(...)),
@@ -43,13 +46,72 @@ struct AssetsFinder {
var_identifiers: HashMap<String, (AssetKind, VarAssetName, VarAssetSchema)>,
}
impl Visitor for AssetsFinder {
impl<'a> Visitor<'a> for AssetsFinder {
fn visit_stmt(&mut self, stmt: &'a Stmt) {
if let Stmt::Assign(node) = stmt {
self.track_assignment(node);
}
// Continue with generic visit to catch any other assets in the statement
walk_stmt(self, stmt);
}
fn visit_expr(&mut self, expr: &'a Expr) {
match expr {
// A detected asset call does not recurse,
// so this is only reached when no further context was found
Expr::StringLiteral(s) => self.visit_str(s.value.to_str()),
Expr::FString(f) => {
// Literal text between two interpolations reads as one string, across the
// pieces of an implicitly concatenated f-string too.
let mut literal = String::new();
for part in &f.value {
match part {
FStringPartRef::Literal(s) => literal.push_str(&s.value),
FStringPartRef::FString(f) => {
for element in &f.elements {
match element {
InterpolatedStringElement::Literal(l) => {
literal.push_str(&l.value)
}
InterpolatedStringElement::Interpolation(_) => {
self.visit_str(&std::mem::take(&mut literal))
}
}
}
}
}
}
self.visit_str(&literal);
walk_expr(self, expr);
}
Expr::Call(node) => {
if self.visit_expr_call_inner(node).is_err() {
walk_expr(self, expr);
}
}
_ => walk_expr(self, expr),
}
}
}
impl AssetsFinder {
fn visit_str(&mut self, s: &str) {
if let Some((kind, path)) = parse_asset_syntax(s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
// Handle assignment statements like: x = wmill.datatable('name')
fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) {
fn track_assignment(&mut self, node: &StmtAssign) {
// Check if the value is a call to a tracked function
if let Some(Expr::Name(expr_name)) = node.targets.first() {
// Remove any tracked variables with that name in case of reassignment
let Ok(var_name) = expr_name.id.parse::<String>();
let var_name = expr_name.id.to_string();
let removed = self.var_identifiers.remove(&var_name);
// if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)),
// we still want to register the asset as unknown access type
@@ -69,59 +131,12 @@ impl Visitor for AssetsFinder {
if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) {
// Track target variable
let Ok(var_name) = expr_name.id.parse::<String>();
self.var_identifiers
.insert(var_name, (kind.clone(), name.clone(), schema.clone()));
}
}
// Continue with generic visit to catch any other assets in the expression
self.generic_visit_stmt_assign(node);
}
// visit_call_expr will not recurse if it detects an asset,
// so this will only be called when no further context was found
fn visit_expr_constant(&mut self, node: ExprConstant) {
match node.value {
Constant::Str(s) => {
if let Some((kind, path)) = parse_asset_syntax(&s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
_ => self.generic_visit_expr_constant(node),
}
}
fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) {
match self.visit_expr_call_inner(&node) {
Ok(_) => {}
Err(_) => {
// Check keyword arguments for assets before falling back to generic visit
for keyword in &node.keywords {
if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) =
&keyword.value
{
if let Some((kind, path)) = parse_asset_syntax(s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
}
self.generic_visit_expr_call(node);
}
}
}
}
impl AssetsFinder {
/// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc.
fn extract_asset_from_call(
&self,
@@ -147,18 +162,20 @@ impl AssetsFinder {
// Get the first argument (the asset name)
let name = call
.arguments
.args
.first()
.or_else(|| {
// Try keyword argument 'name' or 'path'
call.keywords
call.arguments
.keywords
.iter()
.find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path")))
.map(|kw| &kw.value)
})
.and_then(|first_arg| {
if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg {
Some(name.clone())
if let Expr::StringLiteral(name) = first_arg {
Some(name.value.to_str().to_string())
} else {
None
}
@@ -179,44 +196,39 @@ impl AssetsFinder {
Some((kind, name, schema))
}
fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> {
fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> {
let ident: String = node
.func
.as_name_expr()
.and_then(|o| o.id.parse().ok())
.map(|o| o.id.to_string())
.or_else(|| {
node.func
.as_attribute_expr()
.and_then(|attr| attr.attr.parse().ok())
.map(|attr| attr.attr.to_string())
})
.ok_or(())?;
// `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()`
let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute {
value, ..
}) = node.func.as_ref()
{
if let Expr::Name(expr_name) = value.as_ref() {
expr_name.id.parse().map_err(|_| ())?
} else {
return Err(());
}
} else {
return Err(());
};
let obj_name: String = node
.func
.as_attribute_expr()
.and_then(|attr| attr.value.as_name_expr())
.map(|name| name.id.to_string())
.ok_or(())?;
if obj_name == "wmill" {
// Continue
} else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) {
if ident == "query" {
let expr_name = node.args.get(0).or_else(|| {
node.keywords
let expr_name = node.arguments.args.get(0).or_else(|| {
node.arguments
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some("name"))
.map(|kw| &kw.value)
});
let sql = match expr_name {
Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql,
Some(Expr::StringLiteral(sql)) => sql.value.to_str(),
_ => return Err(()),
};
// We use the SQL parser to detect RW, specific tables, etc.
@@ -224,7 +236,7 @@ impl AssetsFinder {
*kind,
path,
schema.as_deref(),
&sql,
sql,
);
match sql_assets {
Ok(Some(sql_assets)) => self.assets.extend(sql_assets),
@@ -252,9 +264,10 @@ impl AssetsFinder {
};
let arg_val = match arg {
Arg(i, name) => node.args.get(i).or_else(|| {
Arg(i, name) => node.arguments.args.get(i).or_else(|| {
// Get arg by name
node.keywords
node.arguments
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.map(|kw| &kw.value)
@@ -270,8 +283,9 @@ impl AssetsFinder {
self.assets
.push(ParseAssetsResult { kind, path, access_type, columns: None });
}
Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => {
let path = parse_asset_syntax(&value, false)
Some(Expr::StringLiteral(value)) => {
let value = value.value.to_str();
let path = parse_asset_syntax(value, false)
.map(|(_, p)| p)
.unwrap_or(std::borrow::Cow::Borrowed(value));
self.assets.push(ParseAssetsResult {
@@ -291,26 +305,24 @@ impl AssetsFinder {
struct Arg(usize, &'static str);
/// Extract a string-literal keyword argument, e.g. `s3="value"` in a call.
fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option<String> {
fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option<String> {
keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.and_then(|kw| match &kw.value {
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()),
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
}
/// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`.
fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String> {
dict.keys
fn dict_str_value(dict: &ExprDict, name: &str) -> Option<String> {
dict.items
.iter()
.zip(dict.values.iter())
.find_map(|(key, value)| match (key.as_ref()?, value) {
(
Expr::Constant(ExprConstant { value: Constant::Str(k), .. }),
Expr::Constant(ExprConstant { value: Constant::Str(v), .. }),
) if k.as_str() == name => Some(v.clone()),
.find_map(|item| match (item.key.as_ref()?, &item.value) {
(Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => {
Some(v.value.to_str().to_string())
}
_ => None,
})
}
@@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String>
/// matches the TS object form and the `# on s3:///…` trigger form exactly.
fn s3_object_arg_path(expr: &Expr) -> Option<String> {
let uri = match expr {
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
Expr::StringLiteral(s) => {
let s = s.value.to_str();
match s
.strip_prefix("s3://")
.and_then(|rest| rest.split_once('/'))
{
Some((_, key)) if !key.is_empty() => s.clone(),
Some((_, key)) if !key.is_empty() => s.to_string(),
_ => return None,
}
}
@@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option<String> {
if func_name != "S3Object" {
return None;
}
let key = keyword_str_value(&call.keywords, "s3")?;
let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default();
let key = keyword_str_value(&call.arguments.keywords, "s3")?;
let storage =
keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default();
format!("s3://{storage}/{key}")
}
Expr::Dict(dict) => {
@@ -822,4 +836,62 @@ def main():
},])
);
}
#[test]
fn test_py_asset_parser_recent_syntax() {
let input = r#"
import wmill
lazy import polars
type Rows = list[dict]
def main[T](name: str):
try:
with wmill.load_s3_file_reader('s3:///in.csv') as f:
f.read()
except ValueError, TypeError:
pass
greeting = t"hello {name}"
return f"{wmill.get_resource("u/user/pg")["host"]}"
"#;
let s = parse_assets(input).map(|o| o.assets);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/in.csv".to_string(),
access_type: Some(R),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::Resource,
path: "u/user/pg".to_string(),
access_type: None,
columns: None,
},
])
);
}
#[test]
fn test_py_asset_parser_schema_qualified_db_without_table() {
let input = r#"
import wmill
def main():
db = wmill.datatable('dt:sch')
db.query("ATTACH 'ducklake://other' AS o")
db.query("SELECT * FROM t")
"#;
let s = parse_assets(input).map(|o| o.assets);
assert_eq!(
s.map_err(|e| e.to_string())
.map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::<Vec<_>>()),
Ok(vec![
(AssetKind::DataTable, "dt".to_string()),
(AssetKind::DataTable, "dt/sch.t".to_string()),
(AssetKind::Ducklake, "other".to_string()),
])
);
}
}
backend/parsers/windmill-parser-py-imports/Cargo.toml
+2 -1
View File
@@ -23,7 +23,8 @@ tracing.workspace = true
[dependencies]
windmill-parser.workspace = true
rustpython-parser.workspace = true
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
phf.workspace = true
itertools.workspace = true
serde_json.workspace = true
backend/parsers/windmill-parser-py-imports/src/lib.rs
+46 -37
View File
@@ -13,9 +13,9 @@ use async_recursion::async_recursion;
use itertools::Itertools;
use lazy_static::lazy_static;
#[cfg(not(target_arch = "wasm32"))]
use std::str::FromStr;
#[cfg(not(target_arch = "wasm32"))]
use std::collections::HashMap;
#[cfg(not(target_arch = "wasm32"))]
use std::str::FromStr;
use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP};
#[cfg(not(target_arch = "wasm32"))]
@@ -23,11 +23,8 @@ use regex::Regex;
#[cfg(target_arch = "wasm32")]
use regex_lite::Regex;
use rustpython_parser::{
ast::{Stmt, StmtImport, StmtImportFrom, Suite},
text_size::TextRange,
Parse,
};
use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom};
use ruff_python_parser::parse_module;
#[cfg(not(target_arch = "wasm32"))]
use sqlx::{Pool, Postgres};
#[cfg(not(target_arch = "wasm32"))]
@@ -160,6 +157,7 @@ pub struct ImportPin {
}
pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImport>> {
let full_code = code;
// Use regex to safely find the main function definition
let mut code = DEF_MAIN_RE
.split(code)
@@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
// This is needed because we've split off the real main function above
let code_with_fake_main = format!("{}\n\ndef main(): pass", code);
let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| {
anyhow::anyhow!("Error parsing code for imports: {}", e.to_string())
})?;
let (code, ast) = match parse_module(&code_with_fake_main) {
Ok(parsed) => (code, parsed.into_suite()),
// The split lands on any line reading like the definition of `main`, one inside a
// string included: what precedes it is then not a module, while the whole file is.
Err(e) => match parse_module(full_code) {
Ok(parsed) => (full_code.to_string(), parsed.into_suite()),
Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()),
},
};
// Note: We're still using the original code for finding pins,
// as the TextRange values from the parsed AST would be based on code_with_fake_main
// but we want to match against the original code
let find_pin = |range: TextRange, key: String| {
let find_pin = |stmt_end: usize, key: String| {
let hs = code
.chars()
.skip(range.end().to_usize())
.skip(stmt_end)
.take_while(|e| *e != '\n')
.collect::<String>();
@@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
let mut nimports: Vec<NImport> = ast
.into_iter()
.filter_map(|x| match x {
Stmt::Import(StmtImport { names, range }) => names
Stmt::Import(StmtImport { names, range, .. }) => names
.get(0)
.and_then(|al| find_pin(range, al.name.to_string()))
.and_then(|al| find_pin(range.end().to_usize(), al.name.to_string()))
.or(Some(
names
.into_iter()
@@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
.flatten()
.collect::<Vec<NImport>>(),
)),
Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
Some(process_import(
module.map(|x| x.to_string()),
path,
i.to_usize(),
))
}
Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some(
process_import(module.map(|x| x.to_string()), path, level as usize),
),
Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin(
range,
range.end().to_usize(),
module.clone().map(|x| x.to_string()).unwrap_or_default(),
)
.or(Some(process_import(module.map(|x| x.to_string()), path, 0))),
@@ -351,13 +351,16 @@ async fn parse_python_imports_inner(
temp_script_refs: &Option<HashMap<String, String>>,
) -> error::Result<HashMap<String, NImportResolved>> {
tracing::debug!("Parsing python imports for path: {}", path);
let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
let PythonAnnotations { py310, py311, py312, py313, py314, py315, .. } =
PythonAnnotations::parse(&code);
tracing::debug!(
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}",
"Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}, py314: {}, py315: {}",
py310,
py311,
py312,
py313
py313,
py314,
py315
);
let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> {
@@ -373,6 +376,8 @@ async fn parse_python_imports_inner(
push_version_specifiers(py311, "==3.11.*".to_owned())?;
push_version_specifiers(py312, "==3.12.*".to_owned())?;
push_version_specifiers(py313, "==3.13.*".to_owned())?;
push_version_specifiers(py314, "==3.14.*".to_owned())?;
push_version_specifiers(py315, "==3.15.*".to_owned())?;
for x in code.lines() {
if x.starts_with("# py:") || x.starts_with("#py:") {
@@ -513,8 +518,14 @@ async fn parse_python_imports_inner(
let mut nested = match n {
NImport::Relative(rpath) => {
// First try to get content from temp_script_refs cache if available
let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) {
tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash);
let code_from_cache = if let Some(hash) =
temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath))
{
tracing::debug!(
"Found relative import '{}' in temp_script_refs with hash '{}'",
rpath,
hash
);
match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await {
Ok(content) => Some(content),
Err(e) => {
@@ -529,19 +540,17 @@ async fn parse_python_imports_inner(
// Use cached content if available, otherwise fall back to deployed script
let code = match code_from_cache {
Some(content) => content,
None => {
sqlx::query_scalar!(
r#"
None => sqlx::query_scalar!(
r#"
SELECT content FROM script WHERE path = $1 AND workspace_id = $2
AND archived = false ORDER BY created_at DESC LIMIT 1
"#,
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string())
}
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string()),
};
if already_visited.contains(&rpath) {
backend/parsers/windmill-parser-py-imports/src/mapping.rs
+5
View File
@@ -383,5 +383,10 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! {
"vt" => "vt-py",
"grpc" => "grpcio",
"serpapi" => "google-search-results",
"pptx" => "python-pptx",
"OpenSSL" => "pyOpenSSL",
"win32api" => "pywin32",
"ruamel" => "ruamel.yaml",
"speech-recognition" => "SpeechRecognition",
// Add new entry here ^
};
backend/parsers/windmill-parser-py-imports/tests/tests.rs
+86 -1
View File
@@ -1,7 +1,9 @@
mod tests {
use sqlx::{Pool, Postgres};
use windmill_parser_py_imports::parse_python_imports;
use windmill_parser_py_imports::{
parse_code_for_imports, parse_python_imports, ImportPin, NImport,
};
#[sqlx::test(fixtures("base"))]
async fn test_parse_python_imports(db: Pool<Postgres>) -> anyhow::Result<()> {
@@ -117,4 +119,87 @@ def main():
Ok(())
}
fn auto(pkg: &str) -> NImport {
NImport::Auto { pkg: pkg.to_string(), key: None }
}
#[test]
fn test_parse_lazy_imports() -> anyhow::Result<()> {
let code = "
lazy import requests
lazy import numpy # pin: numpy==2.1.0
lazy from pandas import DataFrame
lazy from f.shared import helper
lazy = 1
def main():
pass
";
assert_eq!(
parse_code_for_imports(code, "f/foo/bar")?,
vec![
NImport::Pin {
pins: vec![ImportPin {
pkg: "numpy==2.1.0".to_string(),
path: "f/foo/bar".to_string()
}],
key: "numpy".to_string()
},
auto("pandas"),
auto("requests"),
NImport::Relative("f/shared".to_string()),
]
);
Ok(())
}
#[test]
fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> {
let code = r#"
import requests
type Alias = list[int]
NAME = "x"
D = {"a": 1}
L = [[1], [2]]
TEMPLATE = t"hello {NAME}"
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
PEP798 = [*a for a in L]
try:
import ujson
except ImportError, ValueError:
pass
def main():
pass
"#;
assert_eq!(
parse_code_for_imports(code, "f/foo/bar")?,
vec![auto("requests")]
);
Ok(())
}
#[test]
fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> {
for example in ["def main[T](x: T):", "def main(x):"] {
let code = format!(
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n"
);
assert_eq!(
parse_code_for_imports(&code, "f/foo/bar")?,
vec![NImport::Pin {
pins: vec![ImportPin {
pkg: "requests==2.32.0".to_string(),
path: "f/foo/bar".to_string()
}],
key: "requests".to_string()
}],
"{code}"
);
}
Ok(())
}
}
backend/parsers/windmill-parser-py/Cargo.toml
+2 -2
View File
@@ -10,8 +10,8 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
rustpython-parser.workspace = true
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
itertools.workspace = true
serde_json.workspace = true
anyhow.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
backend/parsers/windmill-parser-py/src/lib.rs
+206 -103
View File
@@ -13,13 +13,11 @@ use itertools::Itertools;
use serde_json::json;
use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ};
use rustpython_parser::{
ast::{
Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt,
StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite,
},
Parse,
use ruff_python_ast::{
Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign,
StmtClassDef, StmtFunctionDef,
};
use ruff_python_parser::parse_module;
pub mod pydantic_parser;
@@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool {
|| code.contains("import dataclasses")
}
/// What follows `def <name>` or `async def <name>` on a line opening that function's
/// parameters or type parameters.
fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> {
let rest = line
.strip_prefix("async def ")
.or_else(|| line.strip_prefix("def "))?
.strip_prefix(name)?;
(rest.starts_with('(') || rest.starts_with('[')).then_some(rest)
}
fn filter_non_main(code: &str, main_name: &str) -> String {
let def_main = format!("def {}(", main_name);
let async_def_main = format!("async def {}(", main_name);
let mut filtered_code = String::new();
let mut code_iter = code.split("\n");
let mut remaining: String = String::new();
while let Some(line) = code_iter.next() {
if line.starts_with(&async_def_main) {
filtered_code += &async_def_main;
remaining += line.strip_prefix(&async_def_main).unwrap();
remaining += "\n";
remaining += &code_iter.join("\n");
break;
} else if line.starts_with(&def_main) {
filtered_code += &def_main;
remaining += line.strip_prefix(&def_main).unwrap();
if let Some(rest) = strip_def(line, main_name) {
filtered_code += &line[..line.len() - rest.len()];
remaining += rest;
remaining += "\n";
remaining += &code_iter.join("\n");
break;
@@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String {
return String::new();
}
let mut chars = remaining.chars();
// PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters.
if remaining.starts_with('[') {
let mut open_brackets = 0;
for c in chars.by_ref() {
if c == '[' {
open_brackets += 1;
} else if c == ']' {
open_brackets -= 1;
}
filtered_code.push(c);
if open_brackets == 0 {
break;
}
}
}
if chars.next() != Some('(') {
return String::new();
}
filtered_code.push('(');
let mut open_parens = 1;
while let Some(c) = chars.next() {
@@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
let filtered_code = filter_relevant_statements(code);
let ast = match Suite::parse(&filtered_code, "main.py") {
Ok(ast) => ast,
let ast = match parse_module(&filtered_code) {
Ok(parsed) => parsed.into_suite(),
Err(_) => return CodeMetadata { enums, descriptions },
};
for stmt in ast {
match stmt {
Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => {
let is_enum = bases.iter().any(|base| {
Stmt::ClassDef(class_def) if has_enum => {
let is_enum = class_def.bases().iter().any(|base| {
matches!(base, Expr::Name(ExprName { id, .. })
if id == "Enum" || id == "IntEnum" || id == "StrEnum"
|| id == "Flag" || id == "IntFlag")
@@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
let mut values = Vec::new();
let mut members = HashMap::new();
let StmtClassDef { name, body, .. } = class_def;
for item in body {
if let Stmt::Assign(StmtAssign { targets, value, .. }) = item {
if let Some(Expr::Name(ExprName { id: target_name, .. })) =
targets.first()
{
if !target_name.starts_with('_') {
if let Expr::Constant(ExprConstant {
value: Constant::Str(val),
..
}) = value.as_ref()
{
if let Expr::StringLiteral(val) = value.as_ref() {
let val = val.value.to_str();
values.push(val.to_string());
members.insert(target_name.to_string(), val.to_string());
}
@@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
}
}
}
Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => {
if &func_name == main_name {
Stmt::FunctionDef(StmtFunctionDef {
name: func_name, body, is_async: false, ..
}) if has_docstring => {
if func_name.as_str() == main_name {
if let Some(Stmt::Expr(expr_stmt)) = body.first() {
if let Expr::Constant(ExprConstant {
value: Constant::Str(docstring),
..
}) = expr_stmt.value.as_ref()
{
descriptions = parse_docstring_args(docstring);
if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() {
descriptions = parse_docstring_args(docstring.value.to_str());
}
}
}
@@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap<String, String> {
descriptions
}
/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring
/// showing an example for instance. When several lines do, only the parsed module tells which
/// one is the function.
fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option<Parameters> {
code.split('\n')
.filter(|line| strip_def(line, main_name).is_some())
.nth(1)?;
parse_module(code)
.ok()?
.into_suite()
.into_iter()
.find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(*parameters)
}
_ => None,
})
}
/// skip_params is a micro optimization for when we just want to find the main
/// function without parsing all the params.
pub fn parse_python_signature(
@@ -310,41 +347,39 @@ pub fn parse_python_signature(
// - If no models: parse only the filtered main function
let (params, module) = if should_parse_for_models(code) {
// Parse full code once for both Pydantic detection and signature extraction
let ast = Suite::parse(code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(code)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
// Extract main function from full AST
let params = ast.iter().find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => {
Some(args.as_ref().clone())
}
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
if name == &main_name =>
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(args.as_ref().clone())
Some(parameters.as_ref().clone())
}
_ => None,
});
// Keep AST for Pydantic/dataclass detection
(params, Some(ast))
} else if let Some(params) = main_params_among_lookalikes(code, &main_name) {
(Some(params), None)
} else {
// No models detected - parse only the filtered main function
let filtered_code = filter_non_main(code, &main_name);
if filtered_code.is_empty() {
(None, None)
} else {
let ast = Suite::parse(&filtered_code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(&filtered_code)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
let params = ast.into_iter().find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => {
Some(*args)
}
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
if &name == &main_name =>
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(*args)
Some(*parameters)
}
_ => None,
});
@@ -378,8 +413,6 @@ pub fn parse_python_signature(
if !skip_params && params.is_some() {
let params = params.unwrap();
let def_arg_start = params.args.len() - params.defaults().count();
// Two-pass approach for lazy metadata extraction:
// Pass 1: Parse types without enum info to determine if metadata is needed
// Pass 2: Re-parse unknown types with metadata only if necessary
@@ -392,11 +425,11 @@ pub fn parse_python_signature(
.iter()
.enumerate()
.map(|(i, x)| {
let arg_name = x.as_arg().arg.to_string();
let arg_name = x.parameter.name.to_string();
let (typ, has_default) = x
.as_arg()
.parameter
.annotation
.as_ref()
.as_deref()
.map_or((Typ::Unknown, false), |e| {
parse_expr(e, &empty_enums, module_ref)
});
@@ -423,21 +456,16 @@ pub fn parse_python_signature(
.into_iter()
.map(|(i, arg_name, mut typ, mut has_default)| {
if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() {
if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() {
if let Some(annotation) = params.args[i].parameter.annotation.as_deref() {
(typ, has_default) =
parse_expr(annotation, &metadata.enums, module_ref);
}
}
let default = if i >= def_arg_start {
params
.defaults()
.nth(i - def_arg_start)
.map(|expr| to_value(expr, &metadata.enums))
.flatten()
} else {
None
};
let default = params.args[i]
.default
.as_deref()
.and_then(|expr| to_value(expr, &metadata.enums));
let should_get_type_from_default = match &typ {
Typ::Unknown => true,
@@ -465,10 +493,10 @@ pub fn parse_python_signature(
// debounce accumulation support. Falls back to docstring
// description if no union array pattern is found.
let union_otyp = params.args[i]
.as_arg()
.parameter
.annotation
.as_ref()
.and_then(|ann| detect_py_union_array_otyp(ann.as_ref()));
.as_deref()
.and_then(detect_py_union_array_otyp);
Arg {
otyp: union_otyp.or_else(|| {
@@ -509,12 +537,8 @@ pub fn parse_python_signature(
}
}
fn parse_expr(
e: &Box<Expr>,
enums: &HashMap<String, EnumInfo>,
module: Option<&[Stmt]>,
) -> (Typ, bool) {
match e.as_ref() {
fn parse_expr(e: &Expr, enums: &HashMap<String, EnumInfo>, module: Option<&[Stmt]>) -> (Typ, bool) {
match e {
Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false),
Expr::Attribute(x) => {
if let Some(name) = x.value.as_name_expr() {
@@ -531,12 +555,9 @@ fn parse_expr(
}
}
Expr::BinOp(x) => {
if matches!(
x.right.as_ref(),
Expr::Constant(ExprConstant { value: Constant::None, .. })
) {
if x.right.is_none_literal_expr() {
(parse_expr(&x.left, enums, module).0, true)
} else if detect_py_union_array_otyp(e.as_ref()).is_some() {
} else if detect_py_union_array_otyp(e).is_some() {
// T | list[T] — parsed type is Unknown; otyp is set separately
(Typ::Unknown, false)
} else {
@@ -552,7 +573,7 @@ fn parse_expr(
.elts
.iter()
.map(|x| match x {
Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()),
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
.filter_map(|x| x)
@@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String {
}
}
fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
fn to_value(et: &Expr, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
match et {
Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)),
Expr::Dict(ExprDict { keys, values, .. }) => {
let v = keys
.into_iter()
.zip(values)
.map(|(k, v)| {
let key = k
Expr::NoneLiteral(_) => Some(json!(null)),
Expr::BooleanLiteral(b) => Some(json!(b.value)),
Expr::StringLiteral(s) => Some(json!(s.value.to_str())),
Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::<Vec<u8>>())),
Expr::NumberLiteral(n) => Some(match &n.value {
Number::Int(i) => {
serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number"))
}
Number::Float(f) => json!(f),
Number::Complex { real, imag } => json!([real, imag]),
}),
Expr::EllipsisLiteral(_) => Some(json!("...")),
Expr::Dict(ExprDict { items, .. }) => {
let v = items
.iter()
.map(|item| {
let key = item
.key
.as_ref()
.map(|e| to_value(e, enums))
.flatten()
@@ -642,7 +674,7 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
_ => None,
})
.unwrap_or_else(|| "no_key".to_string());
(key, to_value(&v, enums))
(key, to_value(&item.value, enums))
})
.collect::<HashMap<String, _>>();
Some(json!(v))
@@ -672,20 +704,6 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
}
}
fn constant_to_value(c: &Constant) -> serde_json::Value {
match c {
Constant::None => json!(null),
Constant::Bool(b) => json!(b),
Constant::Str(s) => json!(s),
Constant::Bytes(b) => json!(b),
Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")),
Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::<Vec<_>>()),
Constant::Float(f) => json!(f),
Constant::Complex { real, imag } => json!([real, imag]),
Constant::Ellipsis => json!("..."),
}
}
#[cfg(test)]
mod tests {
@@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED):
);
Ok(())
}
#[test]
fn test_parse_python_sig_type_params() -> anyhow::Result<()> {
for code in [
"def main[T](x: str, y: int = 3):\n return x\n",
"async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n",
// parsed whole rather than pre-filtered down to `main`
"from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n",
] {
let result = parse_python_signature(code, None, false)?;
assert_eq!(result.auto_kind, None, "{code}");
assert_eq!(
result
.args
.iter()
.map(|a| (a.name.as_str(), &a.typ, &a.default))
.collect::<Vec<_>>(),
vec![
("x", &Typ::Str(None), &None),
("y", &Typ::Int, &Some(json!(3)))
],
"{code}"
);
}
Ok(())
}
// A script with a model class is parsed whole, so syntax anywhere in it reaches the parser.
#[test]
fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> {
let code = r#"
from dataclasses import dataclass
lazy import requests
lazy from pandas import DataFrame
type Alias = list[int]
@dataclass
class Point:
x: int
try:
import a
except ImportError, ValueError:
pass
D = {"a": 1}
L = [[1], [2]]
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
PEP798 = [*a for a in L]
def main(p: Point, name: str = "world"):
return t"hello {name}"
"#;
let result = parse_python_signature(code, None, false)?;
assert_eq!(result.args.len(), 2);
assert_eq!(
result.args[0].typ,
Typ::Object(ObjectType::new(
Some("Point".to_string()),
Some(vec![windmill_parser::ObjectProperty {
key: "x".to_string(),
typ: Box::new(Typ::Int)
}])
))
);
assert_eq!(result.args[1].default, Some(json!("world")));
Ok(())
}
#[test]
fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> {
for example in ["def main[T](x: T):", "def main(x: int, y: int):"] {
let code = format!(
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n"
);
let result = parse_python_signature(&code, None, false)?;
assert_eq!(
result.args.iter().map(|a| &a.name).collect::<Vec<_>>(),
vec!["value"],
"{code}"
);
}
Ok(())
}
}
backend/parsers/windmill-parser-py/src/pydantic_parser.rs
+9 -9
View File
@@ -11,8 +11,8 @@
//! This module provides functionality to detect and parse Pydantic models and Python
//! dataclasses from Python AST, enabling automatic UI generation for complex data structures.
use rustpython_parser::ast::{
Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign,
use ruff_python_ast::{
Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign,
};
use std::collections::HashSet;
use windmill_parser::{ObjectProperty, ObjectType, Typ};
@@ -64,7 +64,7 @@ fn detect_model_type_impl(
// Mark as being visited
visited.insert(class_name.to_string());
let result = if is_pydantic_base(&class_def.bases) {
let result = if is_pydantic_base(class_def.bases()) {
// Pydantic BaseModel
parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited)
} else if has_dataclass_decorator(&class_def.decorator_list)
@@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool {
}
/// Checks if a class has @dataclass decorator (standard library)
fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
fn has_dataclass_decorator(decorators: &[Decorator]) -> bool {
for decorator in decorators {
match decorator {
match &decorator.expression {
// Match: @dataclass
Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => {
return true;
@@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
}
/// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2)
fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool {
fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool {
for decorator in decorators {
match decorator {
match &decorator.expression {
// Match: @pydantic.dataclasses.dataclass
Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => {
if let Expr::Attribute(ExprAttribute {
@@ -350,10 +350,10 @@ fn extract_field_type(
// String annotations: "ForwardRef" (forward references)
// Not fully supported - return Unknown with warning
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
Expr::StringLiteral(s) => {
eprintln!(
"Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown",
forward_ref = s,
forward_ref = s.value.to_str(),
);
Typ::Unknown
}
backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs
+10 -7
View File
@@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets(
Ok(mut sql_assets) => {
if let Some(schema) = schema {
for asset in &mut sql_assets.assets {
if asset.kind == kind && asset.path.starts_with(asset_name) {
asset.path = format!(
"{}/{}.{}",
asset_name,
schema,
&asset.path[asset_name.len() + 1..]
);
if asset.kind != kind {
continue;
}
// A statement can name the database alone, with no table to qualify.
let table = asset
.path
.strip_prefix(asset_name)
.and_then(|rest| rest.strip_prefix('/'));
if let Some(table) = table {
asset.path = format!("{asset_name}/{schema}.{table}");
}
}
}
backend/parsers/windmill-parser-wac/Cargo.toml
+3 -2
View File
@@ -9,8 +9,9 @@ name = "windmill_parser_wac"
path = "./src/lib.rs"
[dependencies]
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
ruff_text_size.workspace = true
swc_common.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
backend/parsers/windmill-parser-wac/src/python.rs
+135 -190
View File
@@ -1,12 +1,12 @@
use std::collections::HashMap;
use rustpython_parser::{
ast::{
Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry,
StmtTryStar, StmtWhile,
},
Parse,
use ruff_python_ast::{
CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName,
Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry,
StmtWhile,
};
use ruff_python_parser::parse_module;
use ruff_text_size::{Ranged, TextSize};
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
use crate::validation::{self, CompileError};
@@ -42,9 +42,9 @@ type TaskFunctions = HashMap<String, Option<String>>;
fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
let mut tasks = HashMap::new();
for stmt in stmts {
if let Stmt::AsyncFunctionDef(func) = stmt {
if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt {
for dec in &func.decorator_list {
match dec {
match &dec.expression {
// @task (bare decorator)
Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => {
tasks.insert(func.name.to_string(), None);
@@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
if id.as_str() == "task_script" || id.as_str() == "task_flow" {
// Extract the path from the first positional argument
let path = call.args.first().and_then(|arg| {
if let Expr::Constant(c) = arg {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
}
None
let path = call.arguments.args.first().and_then(|arg| match arg {
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
});
// Extract variable name from target
if let Some(Expr::Name(ExprName { id: var_name, .. })) =
@@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
/// Extract the `path=` keyword argument from a `@task(path="...")` call.
fn extract_task_path_kwarg(call: &ExprCall) -> Option<String> {
for kw in &call.keywords {
for kw in &call.arguments.keywords {
if let Some(ref arg) = kw.arg {
if arg.as_str() == "path" {
if let Expr::Constant(c) = &kw.value {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
if let Expr::StringLiteral(s) = &kw.value {
return Some(s.value.to_str().to_string());
}
}
}
@@ -150,10 +144,9 @@ impl WacWalker {
fn line_of_expr(&self, expr: &Expr) -> usize {
let offset = match expr {
Expr::Call(c) => c.range.start().to_usize(),
Expr::Await(a) => a.range.start().to_usize(),
Expr::Attribute(a) => a.range.start().to_usize(),
Expr::Name(n) => n.range.start().to_usize(),
Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => {
expr.start().to_usize()
}
_ => 0,
};
self.line_index.line_of(offset)
@@ -161,17 +154,15 @@ impl WacWalker {
fn line_of_stmt(&self, stmt: &Stmt) -> usize {
let offset = match stmt {
Stmt::If(s) => s.range.start().to_usize(),
Stmt::For(s) => s.range.start().to_usize(),
Stmt::While(s) => s.range.start().to_usize(),
Stmt::Return(s) => s.range.start().to_usize(),
Stmt::Expr(s) => s.range.start().to_usize(),
Stmt::Try(s) => s.range.start().to_usize(),
Stmt::TryStar(s) => s.range.start().to_usize(),
Stmt::Assign(s) => s.range.start().to_usize(),
Stmt::AnnAssign(s) => s.range.start().to_usize(),
Stmt::FunctionDef(s) => s.range.start().to_usize(),
Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(),
Stmt::If(_)
| Stmt::For(_)
| Stmt::While(_)
| Stmt::Return(_)
| Stmt::Expr(_)
| Stmt::Try(_)
| Stmt::Assign(_)
| Stmt::AnnAssign(_)
| Stmt::FunctionDef(_) => stmt.start().to_usize(),
_ => 0,
};
self.line_index.line_of(offset)
@@ -190,9 +181,7 @@ impl WacWalker {
/// Check if an expression is `asyncio.gather(...)` call
fn is_asyncio_gather_call(expr: &Expr) -> bool {
if let Expr::Call(call) = expr {
if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) =
call.func.as_ref()
{
if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() {
if attr.as_str() == "gather" {
if let Expr::Name(ExprName { id, .. }) = value.as_ref() {
return id.as_str() == "asyncio";
@@ -222,20 +211,20 @@ impl WacWalker {
fn expr_to_source(expr: &Expr) -> String {
match expr {
Expr::Compare(c) => {
let left = Self::expr_to_source(&c.left);
if let Some(comparator) = c.comparators.first() {
let left = Self::expr_to_source(&c.operands[0]);
if let Some(comparator) = c.comparators().first() {
let right = Self::expr_to_source(comparator);
let op = match c.ops.first() {
Some(rustpython_parser::ast::CmpOp::Gt) => ">",
Some(rustpython_parser::ast::CmpOp::Lt) => "<",
Some(rustpython_parser::ast::CmpOp::GtE) => ">=",
Some(rustpython_parser::ast::CmpOp::LtE) => "<=",
Some(rustpython_parser::ast::CmpOp::Eq) => "==",
Some(rustpython_parser::ast::CmpOp::NotEq) => "!=",
Some(rustpython_parser::ast::CmpOp::In) => "in",
Some(rustpython_parser::ast::CmpOp::NotIn) => "not in",
Some(rustpython_parser::ast::CmpOp::Is) => "is",
Some(rustpython_parser::ast::CmpOp::IsNot) => "is not",
Some(CmpOp::Gt) => ">",
Some(CmpOp::Lt) => "<",
Some(CmpOp::GtE) => ">=",
Some(CmpOp::LtE) => "<=",
Some(CmpOp::Eq) => "==",
Some(CmpOp::NotEq) => "!=",
Some(CmpOp::In) => "in",
Some(CmpOp::NotIn) => "not in",
Some(CmpOp::Is) => "is",
Some(CmpOp::IsNot) => "is not",
None => "?",
};
format!("{left} {op} {right}")
@@ -253,14 +242,14 @@ impl WacWalker {
format!("{value}.{}", a.attr)
}
Expr::Name(n) => n.id.to_string(),
Expr::Constant(c) => match &c.value {
rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""),
rustpython_parser::ast::Constant::Int(i) => i.to_string(),
rustpython_parser::ast::Constant::Float(f) => f.to_string(),
rustpython_parser::ast::Constant::Bool(b) => b.to_string(),
rustpython_parser::ast::Constant::None => "None".to_string(),
_ => "...".to_string(),
Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()),
Expr::NumberLiteral(n) => match &n.value {
Number::Int(i) => i.to_string(),
Number::Float(f) => f.to_string(),
Number::Complex { .. } => "...".to_string(),
},
Expr::BooleanLiteral(b) => b.value.to_string(),
Expr::NoneLiteral(_) => "None".to_string(),
_ => "...".to_string(),
}
}
@@ -279,8 +268,16 @@ impl WacWalker {
match stmt {
Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value),
Stmt::Assign(a) => self.expr_contains_step(&a.value),
Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::If(s) => {
self.body_contains_step(&s.body)
|| s.elif_else_clauses
.iter()
.any(|clause| self.body_contains_step(&clause.body))
}
// `async for` is not modelled: its body is skipped like any unsupported statement
Stmt::For(s) if !s.is_async => {
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
}
Stmt::While(s) => {
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
}
@@ -289,19 +286,7 @@ impl WacWalker {
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
})
}
Stmt::TryStar(s) => {
self.body_contains_step(&s.body)
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
})
}
Stmt::Return(_) => false,
@@ -319,11 +304,12 @@ impl WacWalker {
match expr {
Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value),
Expr::Call(call) => {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
return true;
}
if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) {
return call.args.iter().any(|a| self.expr_contains_step(a));
if Self::is_asyncio_gather_call(expr) {
return call
.arguments
.args
.iter()
.any(|a| self.expr_contains_step(a));
}
false
}
@@ -370,12 +356,11 @@ impl WacWalker {
Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value),
Stmt::Assign(a) => self.walk_expr_stmt(&a.value),
Stmt::If(if_stmt) => self.walk_if(if_stmt),
Stmt::For(for_stmt) => self.walk_for(for_stmt),
Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt),
Stmt::While(while_stmt) => self.walk_while(while_stmt),
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt),
Stmt::Return(ret) => self.walk_return(ret),
Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => {
Stmt::FunctionDef(_) => {
if self.stmt_contains_step(stmt) {
self.errors.push(validation::error_step_in_nested_function(
self.line_of_stmt(stmt),
@@ -392,7 +377,7 @@ impl WacWalker {
if let Expr::Await(ExprAwait { value, .. }) = expr {
// await task_fn(...)
if let Expr::Call(call) = value.as_ref() {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
if self.is_task_fn_call(value) {
return self.emit_step(call, expr);
}
// Check for SDK-level calls: step(), sleep(), wait_for_approval()
@@ -430,15 +415,12 @@ impl WacWalker {
"step" => {
// step("name", fn) — extract the name from the first string argument
let name = call
.arguments
.args
.first()
.and_then(|arg| {
if let Expr::Constant(c) = arg {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
}
None
.and_then(|arg| match arg {
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
.unwrap_or_else(|| "step".to_string());
let id = self.next_id();
@@ -452,6 +434,7 @@ impl WacWalker {
}
"sleep" => {
let seconds = call
.arguments
.args
.first()
.map(|arg| Self::expr_to_source(arg))
@@ -517,10 +500,10 @@ impl WacWalker {
});
let mut step_ids = Vec::new();
for arg in &gather_call.args {
for arg in &gather_call.arguments.args {
// Each arg should be task_fn(...)
if let Expr::Call(call) = arg {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
if self.is_task_fn_call(arg) {
let (name, script) = self
.extract_step_info_from_task_call(call)
.unwrap_or(("unknown".into(), "unknown".into()));
@@ -553,15 +536,33 @@ impl WacWalker {
}
fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> {
let has_steps_in_body = self.body_contains_step(&if_stmt.body);
let has_steps_in_else = self.body_contains_step(&if_stmt.orelse);
self.walk_if_chain(
if_stmt.start(),
&if_stmt.test,
&if_stmt.body,
&if_stmt.elif_else_clauses,
)
}
/// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it.
fn walk_if_chain(
&mut self,
start: TextSize,
test: &Expr,
body: &[Stmt],
clauses: &[ElifElseClause],
) -> Option<(String, String)> {
let has_steps_in_body = self.body_contains_step(body);
let has_steps_in_else = clauses
.iter()
.any(|clause| self.body_contains_step(&clause.body));
if !has_steps_in_body && !has_steps_in_else {
return None;
}
let line = self.line_index.line_of(if_stmt.range.start().to_usize());
let condition_source = Self::expr_to_source(&if_stmt.test);
let line = self.line_index.line_of(start.to_usize());
let condition_source = Self::expr_to_source(test);
let branch_id = self.next_id();
let branch_node_id = self.add_node(DagNode {
@@ -573,15 +574,19 @@ impl WacWalker {
let mut last_ids = Vec::new();
if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) {
if let Some((true_first, true_last)) = self.walk_body(body) {
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
last_ids.push(true_last);
} else {
last_ids.push(branch_node_id.clone());
}
if !if_stmt.orelse.is_empty() {
if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) {
if let Some((clause, clauses)) = clauses.split_first() {
let orelse = match &clause.test {
Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses),
None => self.walk_body(&clause.body),
};
if let Some((else_first, else_last)) = orelse {
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
last_ids.push(else_last);
} else {
@@ -611,7 +616,7 @@ impl WacWalker {
return None;
}
let line = self.line_index.line_of(for_stmt.range.start().to_usize());
let line = self.line_index.line_of(for_stmt.start().to_usize());
let iter_source = Self::expr_to_source(&for_stmt.iter);
let start_id = self.next_id();
@@ -644,7 +649,7 @@ impl WacWalker {
return None;
}
let line = self.line_index.line_of(while_stmt.range.start().to_usize());
let line = self.line_index.line_of(while_stmt.start().to_usize());
let condition = Self::expr_to_source(&while_stmt.test);
let start_id = self.next_id();
@@ -677,39 +682,14 @@ impl WacWalker {
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
});
if !has_steps {
return None;
}
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
self.emit_try_catch_branch(
&try_stmt.body,
&try_stmt.handlers,
&try_stmt.finalbody,
line,
)
}
fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> {
let has_steps = self.body_contains_step(&try_stmt.body)
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
});
if !has_steps {
return None;
}
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
let line = self.line_index.line_of(try_stmt.start().to_usize());
self.emit_try_catch_branch(
&try_stmt.body,
&try_stmt.handlers,
@@ -721,7 +701,7 @@ impl WacWalker {
fn emit_try_catch_branch(
&mut self,
try_body: &[Stmt],
handlers: &[rustpython_parser::ast::ExceptHandler],
handlers: &[ExceptHandler],
finally_body: &[Stmt],
line: usize,
) -> Option<(String, String)> {
@@ -746,7 +726,7 @@ impl WacWalker {
// Except handlers
for handler in handlers {
match handler {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
ExceptHandler::ExceptHandler(eh) => {
if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) {
self.add_edge(&branch_node_id, &catch_first, Some("except".to_string()));
last_ids.push(catch_last);
@@ -783,7 +763,7 @@ impl WacWalker {
}
fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> {
let line = self.line_index.line_of(ret.range.start().to_usize());
let line = self.line_index.line_of(ret.start().to_usize());
let id = self.next_id();
let node_id = self.add_node(DagNode {
id: id.clone(),
@@ -796,12 +776,12 @@ impl WacWalker {
}
/// Extract workflow function parameters (no longer skips ctx)
fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
fn extract_params(args: &Parameters) -> Vec<Param> {
let mut params = Vec::new();
for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) {
let name = arg_with_default.def.arg.to_string();
let name = arg_with_default.parameter.name.to_string();
let typ = arg_with_default
.def
.parameter
.annotation
.as_ref()
.map(|ann| WacWalker::expr_to_source(ann));
@@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
}
pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
let ast = rustpython_parser::ast::Suite::parse(code, "<workflow>")
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?;
let ast = parse_module(code)
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?
.into_suite();
// First pass: collect @task functions
let task_functions = collect_task_functions(&ast);
// Find the @workflow async def
let workflow_fn = ast.iter().find_map(|stmt| {
if let Stmt::AsyncFunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return Some(func);
let workflow_fns = || {
ast.iter().filter_map(|stmt| match stmt {
Stmt::FunctionDef(func)
if func.decorator_list.iter().any(|dec| {
matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow")
}) =>
{
Some(func)
}
}
// Also check non-async for error reporting
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return None; // Will be reported as not-async below
}
}
None
});
_ => None,
})
};
// Check for non-async workflow function
let non_async_workflow = ast.iter().find_map(|stmt| {
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
let line_index = LineIndex::new(code);
return Some(line_index.line_of(func.range.start().to_usize()));
}
}
None
});
// Find the @workflow async def
let workflow_fn = workflow_fns().find(|func| func.is_async);
// Check for non-async workflow function. The statement starts at its first decorator,
// the name sits on the `def` line the error points at.
let non_async_workflow = workflow_fns()
.find(|func| !func.is_async)
.map(|func| LineIndex::new(code).line_of(func.name.start().to_usize()));
if let Some(line) = non_async_workflow {
if workflow_fn.is_none() {
@@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError
vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }]
})?;
let params = extract_params(&workflow_fn.args);
let params = extract_params(&workflow_fn.parameters);
let source_hash = compute_source_hash(code);
let mut walker = WacWalker::new(code, task_functions);
@@ -894,13 +849,3 @@ fn compute_source_hash(code: &str) -> String {
hasher.update(code.as_bytes());
format!("{:x}", hasher.finalize())
}
trait ToUsize {
fn to_usize(self) -> usize;
}
impl ToUsize for rustpython_parser::text_size::TextSize {
fn to_usize(self) -> usize {
u32::from(self) as usize
}
}
backend/parsers/windmill-parser-wac/tests/python_tests.rs
+74
View File
@@ -363,3 +363,77 @@ async def main(x: str):
));
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return));
}
#[test]
fn test_elif_chain_nests_branches() {
let code = r#"
from wmill import workflow, task
@task
async def a(): ...
@task
async def b(): ...
@task
async def c(): ...
@workflow
async def main(x: int):
if x > 2:
await a()
elif x == 2:
await b()
else:
await c()
"#;
let dag = parse_python_workflow(code).expect("should parse");
let branches: Vec<(&str, usize)> = dag
.nodes
.iter()
.filter_map(|n| match &n.node_type {
DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)),
_ => None,
})
.collect();
assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]);
// The `elif` branch hangs off the `false` edge of the `if` before it
let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id);
assert!(dag
.edges
.iter()
.any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false")));
assert_eq!(
dag.nodes
.iter()
.filter(|n| matches!(n.node_type, DagNodeType::Merge))
.count(),
2
);
}
#[test]
fn test_workflow_with_recent_syntax() {
let code = r#"
from wmill import workflow, task
lazy import polars
type Rows = list[dict]
@task
async def load[T](rows: T): ...
@workflow
async def main(name: str):
try:
await load(t"hello {name}")
except ValueError, TypeError:
pass
"#;
let dag = parse_python_workflow(code).expect("should parse");
assert!(dag
.nodes
.iter()
.any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load")));
}
backend/parsers/windmill-parser-wasm/Cargo.lock
Generated
+356 -274
View File
File diff suppressed because it is too large Load Diff
backend/parsers/windmill-parser-wasm/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ resolver = "2"
members = ["."]
[workspace.package]
version = "1.828.0"
version = "1.830.0"
edition = "2021"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
backend/src/main.rs
+19 -244
View File
@@ -63,12 +63,14 @@ use windmill_common::{
SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING,
SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING,
SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING,
SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING,
TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING,
UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING,
WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING,
WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING,
WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING,
},
hub_resource_types,
scripts::ScriptLang,
stats_oss::schedule_stats,
triggers::TriggerKind,
@@ -121,7 +123,8 @@ use windmill_worker::{
BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS,
DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR,
POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR,
RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
PY314_CACHE_DIR, PY315_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR,
TAR_JAVA_CACHE_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
@@ -401,111 +404,16 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
Ok(())
}
/// Raw resource type from hub API (schema is a JSON string)
#[derive(serde::Deserialize)]
struct HubResourceTypeRaw {
pub id: i64,
pub name: String,
pub schema: Option<String>,
pub app: String,
pub description: Option<String>,
/// Absent from hubs predating the column, and from caches written before it.
#[serde(default)]
pub format_extension: Option<String>,
/// Doubly optional, so a hub predating the field (no key) is told apart from a type the
/// hub leaves unnamed (null).
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option"
)]
pub display_name: Option<Option<String>>,
}
/// Processed resource type with parsed schema
#[derive(serde::Deserialize, serde::Serialize, Clone)]
pub struct HubResourceType {
pub id: i64,
pub name: String,
pub schema: Option<serde_json::Value>,
pub app: String,
pub description: Option<String>,
/// Doubly optional on purpose. A cache written before this column has no key at
/// all (`None`) and must leave the stored extension alone; one written since
/// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely
/// dropping it and must clear. A single `Option` conflates the two, and picking
/// either meaning breaks the other — as does plain serde, which folds `null`
/// into the outer `None`, hence the wrapping deserializer.
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option",
skip_serializing_if = "Option::is_none"
)]
pub format_extension: Option<Option<String>>,
/// Doubly optional like `format_extension`: a cache written before the field leaves the
/// stored name alone, while a null from the hub clears it.
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option",
skip_serializing_if = "Option::is_none"
)]
pub display_name: Option<Option<String>>,
}
const HUB_RT_CACHE_FILE: &str = "resource_types.json";
async fn cache_hub_resource_types() -> anyhow::Result<()> {
println!("Caching resource types from hub...");
let response = HTTP_CLIENT
.get(format!("{}/resource_types/list", DEFAULT_HUB_BASE_URL))
.header("Accept", "application/json")
.send()
.await
.with_context(|| "Failed to fetch resource types from hub")?;
if !response.status().is_success() {
anyhow::bail!(
"Failed to fetch resource types from hub: {}",
response.status()
);
}
let raw_types: Vec<HubResourceTypeRaw> = response
.json::<Vec<HubResourceTypeRaw>>()
.await
.with_context(|| "Failed to parse resource types from hub")?;
// Parse schema strings into JSON values
let resource_types: Vec<HubResourceType> = raw_types
.into_iter()
.filter_map(|rt| {
let schema = match rt.schema {
Some(s) => match serde_json::from_str(&s) {
Ok(v) => Some(v),
Err(e) => {
println!("Warning: failed to parse schema for {}: {}", rt.name, e);
return None;
}
},
None => None,
};
Some(HubResourceType {
id: rt.id,
name: rt.name,
schema,
app: rt.app,
description: rt.description,
format_extension: Some(rt.format_extension),
display_name: rt.display_name,
})
})
.collect();
let resource_types = hub_resource_types::fetch(DEFAULT_HUB_BASE_URL, None, None).await?;
println!("Fetched {} resource types from hub", resource_types.len());
create_dir_all(&*HUB_RT_CACHE_DIR)?;
let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE);
let cache_path = hub_resource_types::cache_path();
let content = serde_json::to_string_pretty(&resource_types)
.with_context(|| "Failed to serialize resource types")?;
@@ -516,147 +424,6 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> {
Ok(())
}
pub async fn sync_cached_resource_types(db: &sqlx::Pool<sqlx::Postgres>) -> anyhow::Result<()> {
let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE);
if tokio::fs::metadata(&cache_path).await.is_err() {
tracing::info!(
"No cached resource types found at {}, skipping sync",
cache_path
);
return Ok(());
}
tracing::info!("Syncing cached resource types to admins workspace...");
let content = tokio::fs::read_to_string(&cache_path)
.await
.with_context(|| format!("Failed to read cache file from {}", cache_path))?;
let cached_types: Vec<HubResourceType> =
serde_json::from_str(&content).with_context(|| "Failed to parse cached resource types")?;
tracing::info!("Found {} cached resource types", cached_types.len());
// Get existing resource types in admins workspace. `format_extension` is part of
// the comparison below, so a type whose only change is gaining or losing it is
// not mistaken for unchanged; `is_fileset` decides whether it may take one.
let existing_types: Vec<(
String,
Option<serde_json::Value>,
Option<String>,
Option<String>,
bool,
Option<String>,
)> = sqlx::query_as(
"SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'",
)
.fetch_all(db)
.await
.with_context(|| "Failed to fetch existing resource types")?;
let existing_map: std::collections::HashMap<
String,
(
Option<serde_json::Value>,
Option<String>,
Option<String>,
bool,
Option<String>,
),
> = existing_types
.into_iter()
.map(
|(name, schema, desc, format_extension, is_fileset, display_name)| {
(
name,
(schema, desc, format_extension, is_fileset, display_name),
)
},
)
.collect();
let mut synced_count = 0;
let mut skipped_count = 0;
for rt in cached_types {
let existing = existing_map.get(&rt.name);
let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false);
let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone());
let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone());
// A fileset is a set of files, so it cannot also be one file. Create, update
// and the manual sync all reject the pair; this writer would otherwise
// persist it onto a same-named local fileset.
//
// A cache with no key at all leaves the stored value alone, so the target is
// what is already there — which is also what makes the comparison below
// agree with the write instead of re-upserting the row on every boot.
let format_extension = if is_fileset {
None
} else {
match &rt.format_extension {
Some(from_cache) => from_cache.clone(),
None => stored_extension.clone(),
}
};
// No key in the cache leaves the stored name alone, as for the extension. So does a name
// too long for the column: one bad entry must not fail the upsert and end the sync.
let display_name = match &rt.display_name {
Some(Some(name)) if name.chars().count() > 100 => {
tracing::warn!(
"Ignoring the display_name of resource type {}: longer than 100 characters",
rt.name
);
stored_display_name.clone()
}
Some(from_cache) => from_cache.clone(),
None => stored_display_name.clone(),
};
if let Some((existing_schema, existing_desc, _, _, _)) = existing {
if existing_schema == &rt.schema
&& existing_desc == &rt.description
&& stored_extension == format_extension
&& stored_display_name == display_name
{
skipped_count += 1;
continue;
}
}
// Insert or update resource type
sqlx::query(
// `format_extension` is resolved above rather than coalesced here: a
// COALESCE could never clear one, so a hub that dropped an extension
// would leave the stale value behind forever.
"INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)
VALUES ('admins', $1, $2, $3, $4, $5, now())
ON CONFLICT (workspace_id, name) DO UPDATE
SET schema = EXCLUDED.schema, description = EXCLUDED.description,
format_extension = EXCLUDED.format_extension,
display_name = EXCLUDED.display_name, edited_at = now()",
)
.bind(&rt.name)
.bind(&rt.schema)
.bind(&rt.description)
.bind(&format_extension)
.bind(&display_name)
.execute(db)
.await
.with_context(|| format!("Failed to upsert resource type {}", rt.name))?;
synced_count += 1;
}
tracing::info!(
"Synced {} resource types to admins workspace ({} skipped as unchanged)",
synced_count,
skipped_count
);
Ok(())
}
fn print_help() {
println!("Windmill - a fast, open-source workflow engine and job runner.");
println!();
@@ -696,7 +463,7 @@ fn print_help() {
println!(" LICENSE_KEY = None (EE only) Enterprise license key (workers require valid key)");
println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup");
println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool");
println!(" SYNC_CACHED_RT = false Sync cached resource types to admins workspace on server start");
println!(" SYNC_CACHED_RT = false Sync the image's cached resource types to the admins workspace on server start");
println!(" HUB_BASE_URL = https://hub.windmill.dev Hub to fetch scripts from in `cache` mode (server/worker use the DB setting instead)");
println!();
println!("Notes:");
@@ -1015,8 +782,10 @@ async fn windmill_main() -> anyhow::Result<()> {
.map(|v| v.to_lowercase() == "true" || v == "1")
.unwrap_or(false)
{
if let Err(e) = sync_cached_resource_types(db).await {
tracing::warn!("Failed to sync cached resource types: {:#}", e);
match hub_resource_types::sync_from_image_cache(db).await {
Ok(Some(outcome)) => tracing::info!("{}", outcome.summary()),
Ok(None) => {}
Err(e) => tracing::warn!("Failed to sync cached resource types: {:#}", e),
}
}
}
@@ -1748,6 +1517,7 @@ Windmill Community Edition {GIT_VERSION}
if server_mode {
if let Some(db) = conn.as_sql() {
schedule_stats(&db, &HTTP_CLIENT).await;
hub_resource_types::spawn_daily_sync(db.clone(), killpill_tx.subscribe());
}
}
@@ -2322,6 +2092,9 @@ async fn process_notify_event(
RESTART_COORDINATION_SETTING => {
// Internal coordination key for staggered restarts, no action needed
}
SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING => {
// Read by the daily resource type sync each time it runs
}
"plain_emails_telemetry" => {
let enabled = sqlx::query_scalar!(
"SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'"
@@ -2407,6 +2180,8 @@ pub async fn run_workers(
&*PY311_CACHE_DIR,
&*PY312_CACHE_DIR,
&*PY313_CACHE_DIR,
&*PY314_CACHE_DIR,
&*PY315_CACHE_DIR,
&*BUN_BUNDLE_CACHE_DIR,
&*GO_CACHE_DIR,
&*GO_BIN_CACHE_DIR,
backend/src/monitor.rs
+14
View File
@@ -4136,6 +4136,20 @@ pub async fn monitor_db(
}
_ => {}
}
match windmill_common::runnable_job_stats::cleanup_old_runnable_job_stats(
db,
windmill_common::runnable_job_stats::RETENTION_DAYS,
)
.await
{
Ok(count) if count > 0 => {
tracing::info!("Deleted {} old runnable job stats rows", count);
}
Err(e) => {
tracing::error!("Error cleaning up runnable job stats: {:?}", e);
}
_ => {}
}
}
}
};
backend/summarized_schema.txt
+1
View File
@@ -180,6 +180,7 @@ resource_type: workspace_id(char), name(char), schema(jsonb), description(text),
FK: (workspace_id) -> workspace(id)
resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool)
FK: (flow) -> v2_job_queue(id)
runnable_job_stats: hour(bigint), workspace_id(char), runnable_path(char), worker_group(text), job_count(int), total_duration_ms(bigint), max_memory_peak(int), sum_memory_peak(bigint), memory_sample_count(int), total_cpu_ms(bigint)
runnable_settings: hash(bigint), debouncing_settings(bigint), concurrency_settings(bigint)
schedule: workspace_id(char), path(char), edited_by(char), edited_at(ts), schedule(char), enabled(bool), script_path(char), args(jsonb), extra_perms(jsonb), is_flow(bool), email(char), error(text), timezone(char), on_failure(char), on_recovery(char), on_failure_times(int), on_failure_exact(bool), on_failure_extra_args(jsonb), on_recovery_times(int), on_recovery_extra_args(jsonb), ws_error_handler_muted(bool), retry(jsonb), summary(char), no_flow_overlap(bool), tag(char), paused_until(ts), on_success(char), on_success_extra_args(jsonb), cron_version(text), description(text), dynamic_skip(char), labels(text[])
FK: (workspace_id) -> workspace(id)
backend/tests/ai_decision.rs
+1 -1
View File
@@ -117,7 +117,7 @@ async fn test_ai_decision_answers_route_a_following_branch(
"expr": "results.d.output.intent.choice === 'refund'",
"modules": [{"id": "r", "value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(choice){ return 'handled ' + choice }",
"input_transforms": {"choice": {
"type": "javascript",
backend/tests/debounce_e2e.rs
+1 -1
View File
@@ -33,7 +33,7 @@ mod debounce_e2e {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export async function main(items: any[]) { return items }",
"input_transforms": {
"items": { "type": "javascript", "expr": "flow_input.items" }
backend/tests/drafts_delete_for_user.rs
+85
View File
@@ -0,0 +1,85 @@
//! Only a workspace admin can delete a draft that belongs to another user.
//!
//! A discard through `drafts/update` is scoped to the caller's own row, so it
//! never reaches someone else's draft. `drafts/delete_for_user` is the admin
//! route for that, and it must stay closed to everyone else.
use serde_json::{json, Value};
use sqlx::{Pool, Postgres};
use windmill_test_utils::*;
#[sqlx::test(fixtures("base"))]
async fn test_only_admin_deletes_another_users_draft(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let base = format!("http://localhost:{port}/api/w/test-workspace/drafts");
let client = reqwest::Client::new();
let path = "script/u/test-user-2/theirs";
let save = |token: &'static str| {
client
.post(format!("{base}/update/{path}"))
.header("Authorization", format!("Bearer {token}"))
.json(&json!({ "value": { "path": "u/test-user-2/theirs", "content": "" } }))
.send()
};
let resp = save("SECRET_TOKEN_2").await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
let delete = |token: &'static str| {
client
.post(format!(
"{base}/delete_for_user/{path}?username=test-user-2"
))
.header("Authorization", format!("Bearer {token}"))
.send()
};
let own_draft = || async {
client
.get(format!("{base}/get_own/{path}"))
.header("Authorization", "Bearer SECRET_TOKEN_2")
.send()
.await?
.json::<Value>()
.await
};
let resp = delete("SECRET_TOKEN_3").await?;
assert_eq!(resp.status(), 403, "{}", resp.text().await?);
assert!(
!own_draft().await?.is_null(),
"a non-admin deleted the draft"
);
let resp = delete("SECRET_TOKEN").await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
assert!(own_draft().await?.is_null(), "the draft is still there");
let resp = delete("SECRET_TOKEN").await?;
assert_eq!(resp.status(), 404, "{}", resp.text().await?);
// Without a username the delete takes every draft at the path.
for token in ["SECRET_TOKEN_2", "SECRET_TOKEN"] {
let resp = save(token).await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
}
let delete_all = |token: &'static str| {
client
.post(format!("{base}/delete_for_user/{path}"))
.header("Authorization", format!("Bearer {token}"))
.send()
};
let resp = delete_all("SECRET_TOKEN_3").await?;
assert_eq!(resp.status(), 403, "{}", resp.text().await?);
let resp = delete_all("SECRET_TOKEN").await?;
assert_eq!(resp.status(), 200, "{}", resp.text().await?);
let left: i64 = sqlx::query_scalar(
"SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user-2/theirs'",
)
.fetch_one(&db)
.await?;
assert_eq!(left, 0, "drafts survived a path-wide delete");
Ok(())
}
backend/tests/end_user_email.rs
+116 -82
View File
@@ -1,10 +1,8 @@
//! Tests for WM_END_USER_EMAIL environment variable.
//!
//! These tests verify that WM_END_USER_EMAIL is populated with the authenticated
//! user's email when executing app components.
//!
//! TODO: Add tests for scripts and flows once public execution endpoints are identified.
//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup.
//! user's email when executing app components and when running scripts and flows directly.
//! Only apps support non-workspace-member execution via OptAuthed + token lookup.
use serde_json::json;
use sqlx::{Pool, Postgres};
@@ -27,40 +25,6 @@ fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuil
builder.header("Authorization", format!("Bearer {}", token))
}
// TODO: Script tests - need to identify public execution endpoints for non-workspace-members
// async fn run_script(port: u16, token: &str) -> anyhow::Result<String> {
// let url = format!(
// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email",
// port
// );
// let resp = authed(client().post(&url), token)
// .json(&json!({}))
// .send()
// .await?;
// if !resp.status().is_success() {
// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?);
// }
// Ok(resp.json::<serde_json::Value>().await?
// .as_str().unwrap_or("").to_string())
// }
// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members
// async fn run_flow(port: u16, token: &str) -> anyhow::Result<String> {
// let url = format!(
// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow",
// port
// );
// let resp = authed(client().post(&url), token)
// .json(&json!({}))
// .send()
// .await?;
// if !resp.status().is_success() {
// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?);
// }
// Ok(resp.json::<serde_json::Value>().await?
// .as_str().unwrap_or("").to_string())
// }
/// Create an app with inline script via API
async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> {
let url = format!("http://localhost:{}/api/w/test-workspace/apps/create", port);
@@ -74,8 +38,8 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<
"subgrids": {},
"hiddenInlineScripts": [{
"name": "get_email",
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }",
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }",
"path": "f/test/email_app/get_email"
}]
},
@@ -89,7 +53,7 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<
"one_of_inputs": {}
},
// SHA256 hash of raw_code content for anonymous execution
"rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": {
"rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": {
"static_inputs": {},
"one_of_inputs": {}
}
@@ -120,8 +84,8 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res
"css": "",
"inlineScripts": [{
"name": "get_email",
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }"
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }"
}]
},
"policy": {
@@ -134,7 +98,7 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res
"one_of_inputs": {}
},
// SHA256 hash of raw_code content for anonymous execution
"rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": {
"rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": {
"static_inputs": {},
"one_of_inputs": {}
}
@@ -233,8 +197,8 @@ async fn run_app_inline_script(
"args": {},
"component": "get_email",
"raw_code": {
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }",
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }",
"path": format!("{}/get_email", app_path)
}
});
@@ -270,8 +234,8 @@ async fn run_raw_app_inline_script(
"args": {},
"component": "get_email",
"raw_code": {
"language": "deno",
"content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }"
"language": "bun",
"content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }"
}
});
if force_viewer {
@@ -312,41 +276,111 @@ async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Re
anyhow::bail!("timeout waiting for job result")
}
// TODO: Script tests - need to identify public execution endpoints for non-workspace-members
// #[cfg(feature = "deno_core")]
// #[sqlx::test(fixtures("base", "end_user_email"))]
// async fn test_script_wm_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
// initialize_tracing().await;
// set_jwt_secret().await;
// let server = ApiServer::start(db.clone()).await?;
// let port = server.addr.port();
//
// in_test_worker(Connection::Sql(db.clone()), async move {
// let result = run_script(port, SAME_WS_TOKEN).await?;
// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email");
// Ok::<(), anyhow::Error>(())
// }, port).await?;
//
// Ok(())
// }
async fn run_direct(port: u16, token: &str, route: &str) -> anyhow::Result<String> {
let url = format!(
"http://localhost:{}/api/w/test-workspace/jobs/run/{}",
port, route
);
let resp = authed(client().post(&url), token)
.json(&json!({}))
.send()
.await?;
if !resp.status().is_success() {
anyhow::bail!(
"run {route} failed: {} - {}",
resp.status(),
resp.text().await?
);
}
let job_id = resp.text().await?;
wait_for_job_result(port, token, &job_id).await
}
// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members
// #[cfg(feature = "deno_core")]
// #[sqlx::test(fixtures("base", "end_user_email"))]
// async fn test_flow_wm_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
// initialize_tracing().await;
// set_jwt_secret().await;
// let server = ApiServer::start(db.clone()).await?;
// let port = server.addr.port();
//
// in_test_worker(Connection::Sql(db.clone()), async move {
// let result = run_flow(port, SAME_WS_TOKEN).await?;
// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email");
// Ok::<(), anyhow::Error>(())
// }, port).await?;
//
// Ok(())
// }
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base", "end_user_email"))]
async fn test_direct_run_wm_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
set_jwt_secret().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
in_test_worker(
Connection::Sql(db.clone()),
async move {
let result = run_direct(port, SAME_WS_TOKEN, "p/f/test/get_end_user_email").await?;
assert_eq!(result, SAME_WS_EMAIL, "a script run should see its caller");
let result =
run_direct(port, SAME_WS_TOKEN, "f/f/test/get_end_user_email_flow").await?;
assert_eq!(
result, SAME_WS_EMAIL,
"every flow step should see the flow's caller"
);
Ok::<(), anyhow::Error>(())
},
port,
)
.await?;
Ok(())
}
/// A job's `WM_TOKEN` names the identity the job runs as, which differs from its end user when
/// the job runs on behalf of another account. A run it starts must keep the job's end user.
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base", "end_user_email"))]
async fn test_job_token_run_keeps_end_user_email(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
set_jwt_secret().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let calling_job = uuid::Uuid::new_v4();
sqlx::query(
"INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email)
VALUES ($1, 'test2@windmill.dev', 'test-user-2', false, false, '{}', '{}', 'test-workspace', $2)",
)
.bind(calling_job)
.bind(SAME_WS_EMAIL)
.execute(&db)
.await?;
let wm_token = windmill_common::auth::create_jwt_token(
windmill_common::db::Authed {
email: "test2@windmill.dev".to_string(),
username: "test-user-2".to_string(),
is_admin: false,
is_operator: false,
groups: vec!["all".to_string()],
folders: vec![],
scopes: None,
token_prefix: None,
},
"test-workspace",
3600,
Some(calling_job),
None,
None,
None,
)
.await?;
in_test_worker(
Connection::Sql(db.clone()),
async move {
let result = run_direct(port, &wm_token, "p/f/test/get_end_user_email").await?;
assert_eq!(
result, SAME_WS_EMAIL,
"the calling job's end user, not its run-as identity"
);
Ok::<(), anyhow::Error>(())
},
port,
)
.await?;
Ok(())
}
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base", "end_user_email"))]
backend/tests/error_handler.rs
+76 -9
View File
@@ -104,7 +104,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')
VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\n//bun.lock\n<empty>')
"#,
error_handler_code
)
@@ -118,7 +118,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')
VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\n//bun.lock\n<empty>')
"#,
failing_script_hash,
failing_script_code
@@ -155,7 +155,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
language: ScriptLang::Bun,
priority: None,
apply_preprocessor: false,
concurrency_settings: ConcurrencySettings::default(),
@@ -234,7 +234,7 @@ async fn test_error_handler_muted_on_script(db: Pool<Postgres>) -> anyhow::Resul
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '')
VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>')
"#,
)
.execute(&db)
@@ -245,7 +245,7 @@ async fn test_error_handler_muted_on_script(db: Pool<Postgres>) -> anyhow::Resul
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)
VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true)
VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>', true)
"#,
failing_script_hash,
)
@@ -280,7 +280,7 @@ async fn test_error_handler_muted_on_script(db: Pool<Postgres>) -> anyhow::Resul
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
language: ScriptLang::Bun,
priority: None,
apply_preprocessor: false,
concurrency_settings: ConcurrencySettings::default(),
@@ -331,7 +331,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '')
VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>')
"#,
)
.execute(&db)
@@ -342,7 +342,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'deno', 'script', 'test-user', '{}', '', '', '')
VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n<empty>')
"#,
success_script_hash,
)
@@ -377,7 +377,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
language: ScriptLang::Bun,
priority: None,
apply_preprocessor: false,
concurrency_settings: ConcurrencySettings::default(),
@@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool<Postgres>) -> anyh
Ok(())
}
/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
/// and its failure must reach the workspace error handler like a workspace script's.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_error_handler_triggered_on_hub_script_failure(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::jobs::JobPayload;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let version = "990000002";
let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
tokio::fs::create_dir_all(hub_dir).await?;
tokio::fs::write(
format!("{hub_dir}/{version}"),
r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
)
.await?;
// The workspace handler settings are cached per workspace id across the tests of this
// binary, so this reuses the handler path every other test here configures.
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
)
.execute(&db)
.await?;
sqlx::query(
r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb
WHERE workspace_id = 'test-workspace'"#,
)
.execute(&db)
.await?;
let completed_job = RunJob::from(JobPayload::ScriptHub {
path: format!("hub/{version}/test/fail"),
apply_preprocessor: false,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(!completed_job.success, "Hub script should have failed");
let mut handler_parent = None;
for _ in 0..50 {
handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
"SELECT parent_job FROM v2_job
WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'",
)
.fetch_optional(&db)
.await?;
if handler_parent.is_some() {
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
assert_eq!(
handler_parent,
Some(Some(completed_job.id)),
"Workspace error handler should handle the failed hub script"
);
Ok(())
}
backend/tests/fixtures/end_user_email.sql
Vendored
+4 -4
View File
@@ -40,9 +40,9 @@ VALUES (encode(sha256('NO_WS_TOKEN'::bytea), 'hex'), 'NO_WS_TOKE', 'NO_WS_TOKEN'
INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms)
VALUES (
'test-workspace', 'test-user',
'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }',
'export function main() { return process.env.WM_END_USER_EMAIL || ""; }',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script',
'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'bun', E'{}\n//bun.lock\n<empty>', 'script',
'{"g/all": true}'
);
@@ -51,7 +51,7 @@ INSERT INTO flow (workspace_id, summary, description, path, versions, schema, va
VALUES (
'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}',
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}',
'test-user',
'{"g/all": true}'
);
@@ -60,6 +60,6 @@ INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by)
VALUES (
900002, 'test-workspace', 'f/test/get_end_user_email_flow',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}',
'{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}',
'test-user'
);
backend/tests/fixtures/hello.sql
Vendored
+6 -6
View File
@@ -11,7 +11,7 @@ export function main(world: string) {
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
'',
'',
'f/system/hello', 123412, 'deno', '');
'f/system/hello', 123412, 'bun', E'{}\n//bun.lock\n<empty>');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
@@ -28,7 +28,7 @@ export async function main(foo: string, bar: string) {
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}',
'',
'',
'f/system/hello_with_preprocessor', 123413, 'deno', '');
'f/system/hello_with_preprocessor', 123413, 'bun', E'{}\n//bun.lock\n<empty>');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
@@ -140,7 +140,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche
'f/system/hello_with_nodes_flow',
'{1443253234253454}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
'system'
);
@@ -149,7 +149,7 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b
'test-workspace',
'f/system/hello_with_nodes_flow',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}',
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}',
'system'
);
@@ -160,7 +160,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche
'f/system/hello_with_preprocessor',
'{1443253234253456}',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}',
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
'system'
);
@@ -169,6 +169,6 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b
'test-workspace',
'f/system/hello_with_preprocessor',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}',
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record<string, string>\\n\\t\\t\\tquery: Record<string, string>\\n\\t\\t\\theaders: Record<string, string>\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record<string, string[]>\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}',
'system'
);
backend/tests/fixtures/jobs_read_auth.sql
Vendored
+26
View File
@@ -98,6 +98,32 @@ INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, resu
('18181818-1818-1818-1818-181818181818', 'test-workspace', 1000, 'success'::job_status,
'{"agent": "AGENT_CHAT_RESULT"}');
-- A lock job for flow `f/shared/flow1`, as `wmill generate-metadata` queues it, and the
-- token CI would lock with.
INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES (
encode(sha256('RUN_SCOPED_DEPS_TOKEN'::bytea), 'hex'), 'RUN_DEPS_', 'RUN_SCOPED_DEPS_TOKEN',
'test2@windmill.dev', 'lock token', false,
ARRAY['jobs:run:dependencies:f/shared/*']
);
INSERT INTO public.v2_job (
id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email,
kind, runnable_path, tag, visible_to_owner
) VALUES (
'19191919-1919-1919-1919-191919191919', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'flowdependencies', 'f/shared/flow1', 'dependency', true
), (
-- Queued by a broader token with `*` as its path, which the request body allows.
'20202020-2020-2020-2020-202020202020', 'test-workspace', 'test-user-2',
'2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev',
'dependencies', '*', 'dependency', true
);
INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES
('19191919-1919-1919-1919-191919191919', 'test-workspace', 1000, 'success'::job_status,
'{"lock": "DEPS_RESULT"}'),
('20202020-2020-2020-2020-202020202020', 'test-workspace', 1000, 'success'::job_status,
'{"lock": "WILDCARD_DEPS_RESULT"}');
-- A token pairing an app scope with a run scope, as someone driving an app's components
-- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no
-- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach.
backend/tests/fixtures/schedule.sql
Vendored
+3 -3
View File
@@ -13,7 +13,7 @@ export async function main(fail: boolean = true) {
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}',
'',
'',
'f/system/failing_script', 12349, 'deno', '');
'f/system/failing_script', 12349, 'bun', E'{}\n//bun.lock\n<empty>');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
@@ -26,7 +26,7 @@ export async function main() {
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}',
'',
'',
'f/system/schedule_error_handler', 123410, 'deno', '');
'f/system/schedule_error_handler', 123410, 'bun', E'{}\n//bun.lock\n<empty>');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
@@ -39,7 +39,7 @@ export async function main() {
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}',
'',
'',
'f/system/schedule_recovery_handler', 123411, 'deno', '');
'f/system/schedule_recovery_handler', 123411, 'bun', E'{}\n//bun.lock\n<empty>');
INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES (
'test-workspace',
backend/tests/flow_engine_parity.rs
+69 -69
View File
@@ -85,7 +85,7 @@ async fn test_flow_linear_input_transforms(db: Pool<Postgres>) -> anyhow::Result
modules: vec![
flow_module("a", FlowModuleValue::RawScript {
input_transforms: [static_input("x", 10), static_input("y", 5)].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(x: number, y: number) {
return {sum: x + y, product: x * y, items: [1, 2, 3, 4, 5]};
@@ -105,7 +105,7 @@ export function main(x: number, y: number) {
js_input("filtered", "results.a.items.filter(x => x > 2)"),
js_input("from_flow_input", "flow_input.multiplier * results.a.sum"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(total: number, doubled_items: number[], filtered: number[], from_flow_input: number) {
return {total, doubled_items, filtered, from_flow_input};
@@ -157,7 +157,7 @@ async fn test_flow_forloop_complex_expressions(db: Pool<Postgres>) -> anyhow::Re
"a",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -199,7 +199,7 @@ export function main() {
js_input("index", "flow_input.iter.index"),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(user_name: string, user_score: number, bonus: number, index: number) {
return {name: user_name, final_score: user_score + bonus, position: index};
@@ -261,7 +261,7 @@ async fn test_flow_branchone_conditions(db: Pool<Postgres>) -> anyhow::Result<()
"a",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {status: "premium", score: 95, items: [1, 2, 3]};
@@ -293,7 +293,7 @@ export function main() {
js_input("score_from_a", "results.a.score"),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(discount: number, score_from_a: number) {
return {branch: "premium_high", discount, score_from_a};
@@ -320,7 +320,7 @@ export function main(discount: number, score_from_a: number) {
"premium_low",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {branch: "premium_low", discount: 10};
@@ -344,7 +344,7 @@ export function main() {
"default_branch",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {branch: "default", discount: 0};
@@ -398,7 +398,7 @@ async fn test_flow_previous_result_aggregation(db: Pool<Postgres>) -> anyhow::Re
"a",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {value: 10, items: [1, 2, 3]};
@@ -424,7 +424,7 @@ export function main() {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(prev_value: number, prev_items_sum: number) {
return {value: prev_value * 2, sum: prev_items_sum};
@@ -452,7 +452,7 @@ export function main(prev_value: number, prev_items_sum: number) {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(a_value: number, b_value: number, b_sum: number, combined: number) {
return {a_value, b_value, b_sum, combined};
@@ -504,7 +504,7 @@ async fn test_flow_nested_complexity(db: Pool<Postgres>) -> anyhow::Result<()> {
modules: vec![
flow_module("data", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -541,7 +541,7 @@ export function main() {
js_input("values", "results.data.base_values.map(v => v * flow_input.iter.value.multiplier)"),
js_input("sum", "results.data.base_values.reduce((a, b) => a + b, 0) * flow_input.iter.value.multiplier"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(cat_name: string, multiplier: number, values: number[], sum: number) {
return {category: cat_name, multiplier, computed_values: values, total: sum};
@@ -602,7 +602,7 @@ async fn test_flow_object_transformations(db: Pool<Postgres>) -> anyhow::Result<
modules: vec![
flow_module("source", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -633,7 +633,7 @@ export function main() {
js_input("active_bonus", "results.source.config.activeBonus"),
js_input("all_users", "results.source.users"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(admins: string[], active_count: number, admin_bonus: number, active_bonus: number, all_users: any[]) {
// Compute user_summary and total_bonus in the script since complex expressions in input_transforms
@@ -702,7 +702,7 @@ async fn test_flow_skip_if_expressions(db: Pool<Postgres>) -> anyhow::Result<()>
"check",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {should_skip: true, value: 100};
@@ -722,7 +722,7 @@ export function main() {
"maybe_skipped",
FlowModuleValue::RawScript {
input_transforms: [js_input("input_val", "results.check.value * 2")].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(input_val: number) {
return {processed: input_val, was_run: true};
@@ -750,7 +750,7 @@ export function main(input_val: number) {
js_input("prev", "previous_result"),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(check_val: number, prev: any) {
return {check_val, previous: prev};
@@ -799,7 +799,7 @@ async fn test_flow_template_literals(db: Pool<Postgres>) -> anyhow::Result<()> {
modules: vec![
flow_module("data", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -825,7 +825,7 @@ export function main() {
js_input("upper_name", "results.data.firstName.toUpperCase()"),
js_input("items_formatted", "`Items: ${results.data.items.map(i => i.charAt(0).toUpperCase() + i.slice(1)).join(', ')}`"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(full_name: string, greeting: string, items_str: string, upper_name: string, items_formatted: string) {
return {full_name, greeting, items_str, upper_name, items_formatted};
@@ -873,7 +873,7 @@ async fn test_flow_optional_chaining(db: Pool<Postgres>) -> anyhow::Result<()> {
modules: vec![
flow_module("data", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -905,7 +905,7 @@ export function main() {
js_input("default_city", "results.data.partial_user?.address?.city ?? 'Unknown'"),
js_input("default_country", "results.data.user?.address?.country ?? 'USA'"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(city: string, missing_city: any, null_user_name: any, default_city: string, default_country: string) {
return {city, missing_city, null_user_name, default_city, default_country};
@@ -955,7 +955,7 @@ async fn test_flow_parallel_forloop(db: Pool<Postgres>) -> anyhow::Result<()> {
"data",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {items: [1, 2, 3, 4, 5]};
@@ -990,7 +990,7 @@ export function main() {
js_input("cubed", "flow_input.iter.value.value ** 3"),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(original: number, squared: number, cubed: number) {
return {original, squared, cubed};
@@ -1089,7 +1089,7 @@ async fn test_flow_env_access(db: Pool<Postgres>) -> anyhow::Result<()> {
js_input("retry_count", "flow_env.CONFIG.retries"),
js_input("has_auth", "flow_env.CONFIG.features.includes('auth')"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(env_name: string, is_debug: boolean, timeout_val: number, api_url: string, retry_count: number, has_auth: boolean) {
return {env_name, is_debug, timeout_val, api_url, retry_count, has_auth};
@@ -1158,7 +1158,7 @@ async fn test_flow_input_and_env_combined(db: Pool<Postgres>) -> anyhow::Result<
// Conditional based on both
js_input("multiplier", "flow_env.ENV === 'production' ? flow_input.prod_mult : 1"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(effective_limit: number, env_prefix: string, is_prod: boolean, doubled_input: number, multiplier: number) {
return {effective_limit, env_prefix, is_prod, doubled_input, final_value: doubled_input * multiplier};
@@ -1215,7 +1215,7 @@ async fn test_flow_results_optional_chaining(db: Pool<Postgres>) -> anyhow::Resu
"a",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -1287,7 +1287,7 @@ export function main() {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(
user_name: string,
@@ -1361,7 +1361,7 @@ async fn test_flow_large_integers(db: Pool<Postgres>) -> anyhow::Result<()> {
"a",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -1401,7 +1401,7 @@ export function main() {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(
small: number, i32_max: number, over_i32: number,
@@ -1459,7 +1459,7 @@ async fn test_flow_unicode_emoji(db: Pool<Postgres>) -> anyhow::Result<()> {
modules: vec![
flow_module("a", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -1489,7 +1489,7 @@ export function main() {
js_input("mixed_upper", "results.a.mixed.toUpperCase()"),
js_input("template", "`Welcome: ${results.a.greeting}`"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(
greeting: string, greeting_len: number, simple_str: string, has_world: boolean,
@@ -1544,7 +1544,7 @@ async fn test_flow_complex_array_operations(db: Pool<Postgres>) -> anyhow::Resul
modules: vec![
flow_module("a", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -1585,7 +1585,7 @@ export function main() {
js_input("first_three", "results.a.numbers.slice(0, 3)"),
js_input("last_two", "results.a.numbers.slice(-2)"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(
sorted_asc: number[], sorted_desc: number[], active_names: string[],
@@ -1651,7 +1651,7 @@ async fn test_flow_multiline_expressions(db: Pool<Postgres>) -> anyhow::Result<(
modules: vec![
flow_module("a", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -1710,7 +1710,7 @@ export function main() {
}
"#),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(
subtotal: number, discounted_total: number,
@@ -1768,7 +1768,7 @@ async fn test_flow_spread_with_results(db: Pool<Postgres>) -> anyhow::Result<()>
modules: vec![
flow_module("a", FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -1803,7 +1803,7 @@ export function main() {
// Combining multiple spreads
js_input("combined", "{config: {...results.a.config}, tags: [...results.a.tags], source: 'flow'}"),
].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(
merged_config: any, all_tags: string[], full_user: any,
@@ -1869,7 +1869,7 @@ async fn test_flow_nested_forloop_results_access(db: Pool<Postgres>) -> anyhow::
"a",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {
@@ -1908,7 +1908,7 @@ export function main() {
js_input("multiplier", "results.a.multiplier"), // Access outer step from inside for-loop
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(category: string, multiplier: number) {
return {
@@ -1939,7 +1939,7 @@ export function main(category: string, multiplier: number) {
js_input("original_mult", "results.a.multiplier"), // Access outer step
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(items: any[], category: string, original_mult: number) {
return {
@@ -2013,7 +2013,7 @@ async fn test_flow_results_non_existent_step(db: Pool<Postgres>) -> anyhow::Resu
"a",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return { value: 42 };
@@ -2050,7 +2050,7 @@ export function main() {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(
existing: number,
@@ -2126,7 +2126,7 @@ async fn test_flow_env_skip_if_inside_branchone(db: Pool<Postgres>) -> anyhow::R
"inner",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {ran: true};
@@ -2156,7 +2156,7 @@ export function main() {
"branch_marker",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {marker: "branch-marker"};
@@ -2193,7 +2193,7 @@ export function main() {
"after",
FlowModuleValue::RawScript {
input_transforms: [js_input("prev", "previous_result")].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(prev: any) {
return {prev};
@@ -2258,7 +2258,7 @@ async fn test_flow_env_skip_if_nested_branchone(db: Pool<Postgres>) -> anyhow::R
"leaf",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {ran: true};
@@ -2282,7 +2282,7 @@ export function main() {
"inner_marker",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {marker: "inner"};
@@ -2337,7 +2337,7 @@ export function main() {
"after",
FlowModuleValue::RawScript {
input_transforms: [js_input("prev", "previous_result")].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(prev: any) {
return {prev};
@@ -2414,7 +2414,7 @@ async fn test_flow_env_complex_input_transform_in_branch(db: Pool<Postgres>) ->
"Math.min(flow_env.LIMIT, 10) + flow_env.OFFSET",
)]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(value: number) {
return {value};
@@ -2487,7 +2487,7 @@ async fn test_flow_env_skip_if_in_parallel_forloop(db: Pool<Postgres>) -> anyhow
"marker",
FlowModuleValue::RawScript {
input_transforms: [js_input("i", "flow_input.iter.value")].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(i: number) {
return {marker: i};
@@ -2508,7 +2508,7 @@ export function main(i: number) {
"leaf",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {ran: true};
@@ -2594,7 +2594,7 @@ async fn test_flow_env_imported_flow_uses_own_env(db: Pool<Postgres>) -> anyhow:
"id": "leaf",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(key: string) { return {key}; }",
"input_transforms": {
"key": { "type": "javascript", "expr": "flow_env.KEY" }
@@ -2679,7 +2679,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool<Postgres>) ->
"id": "marker",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return {marker: \"from-imported\"}; }",
"input_transforms": {}
}
@@ -2688,7 +2688,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool<Postgres>) ->
"id": "leaf",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return {ran: true}; }",
"input_transforms": {}
},
@@ -2791,7 +2791,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool<Postgres>) -> a
"id": "marker",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return {marker: \"from-imported-branch\"}; }",
"input_transforms": {}
}
@@ -2800,7 +2800,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool<Postgres>) -> a
"id": "leaf",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return {ran: true}; }",
"input_transforms": {}
},
@@ -2932,13 +2932,13 @@ async fn test_flow_env_marks_sub_flows_only_without_ancestor_env(
"parallel": true,
"modules": [{ "id": "leaf", "value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(key: any) { return key ?? null; }",
"input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } }
}}, { "id": "tail", "value": {
// A single simple step is inlined rather than stored as a flow node.
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(prev: any) { return prev; }",
"input_transforms": { "prev": { "type": "javascript", "expr": "results.leaf" } }
}}]
@@ -3024,7 +3024,7 @@ async fn test_flow_env_in_stop_after_if(db: Pool<Postgres>) -> anyhow::Result<()
"first",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {stage: "first"};
@@ -3052,7 +3052,7 @@ export function main() {
"second",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {stage: "second"};
@@ -3110,7 +3110,7 @@ async fn test_stop_after_if_error_include_result(db: Pool<Postgres>) -> anyhow::
"step",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return { userErrors: ["email taken"], ok: false };
@@ -3200,7 +3200,7 @@ async fn test_flow_env_in_retry_if(db: Pool<Postgres>) -> anyhow::Result<()> {
"fails",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
throw new Error("nope");
@@ -3274,7 +3274,7 @@ async fn test_flow_env_in_stop_after_all_iters_if(db: Pool<Postgres>) -> anyhow:
"iter_step",
FlowModuleValue::RawScript {
input_transforms: [js_input("i", "flow_input.iter.value")].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(i: number) {
return {iter: i};
@@ -3316,7 +3316,7 @@ export function main(i: number) {
"after",
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main() {
return {stage: "after-loop"};
@@ -3375,7 +3375,7 @@ async fn test_stop_after_all_iters_if_error_includes_result(
"iter_step",
FlowModuleValue::RawScript {
input_transforms: [js_input("i", "flow_input.iter.value")].into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"
export function main(i: number) {
return { iter: i };
@@ -3453,8 +3453,8 @@ async fn test_flow_expr_step_tag(db: Pool<Postgres>) -> anyhow::Result<()> {
id,
FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
content: "export function main() { return { lang: 'bun' } }".to_string(),
language: ScriptLang::Bun,
content: "export function main() { return { lang: 'python3' } }".to_string(),
path: None,
lock: None,
tag: tag.map(str::to_string),
@@ -3484,7 +3484,7 @@ async fn test_flow_expr_step_tag(db: Pool<Postgres>) -> anyhow::Result<()> {
.bind(job.id)
.fetch_one(&db)
.await?;
assert_eq!(b_tag, "bun");
assert_eq!(b_tag, "python3");
assert!(!job.success);
let result = job.json_result().unwrap();
backend/tests/job_payload.rs
+21 -18
View File
@@ -54,7 +54,7 @@ mod job_payload {
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
language: ScriptLang::Bun,
priority: None,
apply_preprocessor: false,
job_token_scopes: None,
@@ -85,7 +85,7 @@ mod job_payload {
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
language: ScriptLang::Bun,
priority: None,
apply_preprocessor: true,
concurrency_settings:
@@ -165,7 +165,7 @@ mod job_payload {
let test = || async {
let result = RunJob::from(JobPayload::FlowScript {
id: flow_scripts[0],
language: ScriptLang::Deno,
language: ScriptLang::Bun,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default(),
cache_ttl: None,
@@ -185,7 +185,7 @@ mod job_payload {
let test = || async {
let result = RunJob::from(JobPayload::FlowScript {
id: flow_scripts[1],
language: ScriptLang::Deno,
language: ScriptLang::Bun,
concurrency_settings:
windmill_common::runnable_settings::ConcurrencySettings::default(),
cache_ttl: None,
@@ -263,7 +263,7 @@ mod job_payload {
let result = RunJob::from(JobPayload::Dependencies {
path: "f/system/hello".to_string(),
hash: ScriptHash(123412),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
debouncing_settings: Default::default(),
dedicated_worker: None,
})
@@ -351,7 +351,7 @@ mod job_payload {
console.log(greet)
return greet
}"#,
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" }
}
@@ -407,7 +407,7 @@ mod job_payload {
return greet
}"#
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
})
.run_until_complete(&db, false, port)
.await
@@ -416,7 +416,10 @@ mod job_payload {
assert_eq!(
result,
json!({ "lock": "", "status": "Successful lock file generation" })
json!({
"lock": "{\n \"dependencies\": {}\n}\n//bun.lock\n<empty>",
"status": "Successful lock file generation"
})
);
};
test_for_versions(VERSION_FLAGS.iter().copied(), test).await;
@@ -626,7 +629,7 @@ mod job_payload {
console.log(greet)
return greet
}"#,
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" }
}
@@ -673,7 +676,7 @@ mod job_payload {
"content": r#"export function main(world: string) {
return `Hello ${world}!`;
}"#,
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" }
}
@@ -685,7 +688,7 @@ mod job_payload {
"content": r#"export function main(world: string, a: string) {
return `${a} ${world}!`;
}"#,
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" },
"a": { "type": "javascript", "expr": "results.a" }
@@ -704,7 +707,7 @@ mod job_payload {
"x": { "type": "javascript", "expr": "flow_input.iter.value" }
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": r#"export function main(world: string, b: string, x: string) {
return `${x}: ${b} ${world}!`;
}"#,
@@ -857,7 +860,7 @@ mod job_payload {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" }
},
@@ -874,7 +877,7 @@ mod job_payload {
"id": "inner_first",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" }
},
@@ -884,7 +887,7 @@ mod job_payload {
"id": "inner_second",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" },
"first": { "type": "javascript", "expr": "results.inner_first" }
@@ -1045,7 +1048,7 @@ mod job_payload {
"id": "inner",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"iter_val": { "type": "javascript", "expr": "flow_input.iter.value" },
"tag": { "type": "javascript", "expr": "flow_input.tag" }
@@ -1244,7 +1247,7 @@ mod job_payload {
"id": "leaf",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"tag": { "type": "javascript", "expr": "flow_input.tag" },
"iv": { "type": "javascript", "expr": "flow_input.iter.value" }
@@ -1388,7 +1391,7 @@ mod job_payload {
"id": "leaf",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"tag": { "type": "javascript", "expr": "flow_input.tag" }
},
backend/tests/jobs_read_auth.rs
+38
View File
@@ -476,6 +476,44 @@ async fn test_single_job_read_authorization(db: Pool<Postgres>) -> anyhow::Resul
);
}
// A lock job belongs to `jobs:run:dependencies`: its path is whatever the caller sent,
// so a token scoped to run the flow at that path must not read it.
let path = "completed/get_result/19191919-1919-1919-1919-191919191919";
let (status, body) = get(&base, path, Some("RUN_SCOPED_DEPS_TOKEN")).await;
assert!(
status.is_success() && body.contains("DEPS_RESULT"),
"dependencies-scoped token must read its lock job (got {status}): {body}"
);
let (status, body) = get(&base, path, Some("RUN_SCOPED_TOKEN")).await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"a flow-scoped token must not read a lock job (got {status}): {body}"
);
let (status, body) = get(
&base,
&format!("completed/get_result/{FLOW_JOB}"),
Some("RUN_SCOPED_DEPS_TOKEN"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"dependencies-scoped token must not read a flow run at its path (got {status}): {body}"
);
// A lock job filed at `*` matches no path-scoped grant: the stored path is not a pattern.
let (status, body) = get(
&base,
"completed/get_result/20202020-2020-2020-2020-202020202020",
Some("RUN_SCOPED_DEPS_TOKEN"),
)
.await;
assert_eq!(
status,
reqwest::StatusCode::NOT_FOUND,
"a path-scoped token must not read a lock job filed at `*` (got {status}): {body}"
);
// An `apps:run:<app>` scope is a start grant too: the inline-script component run it
// launched — a kind no `jobs:run` scope can name — stays readable to a token scoped
// to that app, and stays out of reach for one that is only scoped to run jobs.
backend/tests/otel.rs
+26 -1
View File
@@ -176,6 +176,10 @@ async fn test_all_metrics_e2e() {
otel_set_worker_uptime("w-uptime", 3600.0);
otel_set_health_status_phase("healthy");
otel_set_health_db_unresponsive(true);
otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024));
otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35);
otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25);
otel_record_job_memory_peak("python3", 100 * 1024 * 1024);
// ── Histograms ──────────────────────────────────────────────────
@@ -189,7 +193,7 @@ async fn test_all_metrics_e2e() {
let metrics = flush_and_get_metrics(&state);
let names = metric_names(&metrics);
// ── Verify all 21 metric names are present ──────────────────────
// ── Verify all metric names are present ─────────────────────────
let expected = [
"windmill.queue.push_count",
@@ -213,6 +217,15 @@ async fn test_all_metrics_e2e() {
"windmill.worker.uptime",
"windmill.health.status",
"windmill.health.db_unresponsive",
"windmill.worker.memory_usage",
"windmill.worker.windmill_memory_usage",
"windmill.worker.memory_limit",
"windmill.worker.cpu_usage",
"windmill.worker.cpu_periods",
"windmill.worker.cpu_throttled_periods",
"windmill.worker.cpu_throttled_time",
"windmill.worker.occupancy_rate",
"windmill.worker.job_memory_peak",
];
for name in expected {
assert!(
@@ -290,6 +303,18 @@ async fn test_all_metrics_e2e() {
.expect("worker.busy data point with worker=worker-test-1 not found");
assert_eq!(dp.1, 1);
let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap();
let values = gauge_i64_values(m);
let dp = values
.iter()
.find(|(attrs, _)| {
attrs
.iter()
.any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu")
})
.expect("worker.memory_limit data point with worker_group=gpu not found");
assert_eq!(dp.1, 1024);
let m = find_metric(&metrics, "windmill.db.pool.active").unwrap();
assert_eq!(gauge_i64_values(m)[0].1, 5);
let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap();
backend/tests/restart_at_step_api.rs
+1 -1
View File
@@ -217,7 +217,7 @@ async fn test_api_restart_at_step_rejects_parallel_loop(db: Pool<Postgres>) -> a
"id": "inner",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {},
"content": "export function main() { return 'ok' }"
}
backend/tests/retry.rs
+16 -8
View File
@@ -70,12 +70,20 @@ mod retry {
fn inner_step() -> &'static str {
r#"
import { connect } from "node:net";
export async function main(index, port) {
const buf = new Uint8Array([0]);
const sock = await Deno.connect({ port });
await sock.write(new Uint8Array([index]));
if (await sock.read(buf) != 1) throw Error("read");
return buf[0];
return await new Promise((resolve, reject) => {
const sock = connect({ port, host: "127.0.0.1" }, () => {
sock.write(new Uint8Array([index]));
});
sock.on("data", (data) => {
resolve(data[0]);
sock.destroy();
});
sock.on("error", reject);
sock.on("close", () => reject(Error("read")));
});
}
"#
}
@@ -105,7 +113,7 @@ def main(last, port):
"port": { "type": "javascript", "expr": "flow_input.port" },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": inner_step(),
},
}],
@@ -281,7 +289,7 @@ def main(last, port):
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return \"ok\" }",
},
}, {
@@ -293,7 +301,7 @@ def main(last, port):
"port": { "type": "javascript", "expr": "flow_input.port" },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": inner_step(),
},
"retry": { "constant": { "attempts": 1, "seconds": 0 } },
backend/tests/success_handler.rs
+70 -3
View File
@@ -123,7 +123,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')
VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\n//bun.lock\n<empty>')
"#,
success_handler_code
)
@@ -138,7 +138,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
sqlx::query!(
r#"
INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')
VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\n//bun.lock\n<empty>')
"#,
test_script_hash,
test_script_code
@@ -175,7 +175,7 @@ export async function main(path: string, email: string, job_id: string, is_flow:
cache_ttl: None,
cache_ignore_s3_path: None,
dedicated_worker: None,
language: ScriptLang::Deno,
language: ScriptLang::Bun,
priority: None,
apply_preprocessor: false,
concurrency_settings: ConcurrencySettings::default(),
@@ -239,3 +239,70 @@ export async function main(path: string, email: string, job_id: string, is_flow:
Ok(())
}
/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job,
/// and its success must reach the workspace success handler like a workspace script's.
#[cfg(all(feature = "enterprise", feature = "private"))]
#[sqlx::test(fixtures("base"))]
async fn test_success_handler_triggered_on_hub_script_success(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
use windmill_common::jobs::JobPayload;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let version = "990000003";
let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR;
tokio::fs::create_dir_all(hub_dir).await?;
tokio::fs::write(
format!("{hub_dir}/{version}"),
r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#,
)
.await?;
// The workspace handler settings are cached per workspace id across the tests of this
// binary, so this reuses the handler path every other test here configures.
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)
VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')",
)
.execute(&db)
.await?;
sqlx::query(
r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb
WHERE workspace_id = 'test-workspace'"#,
)
.execute(&db)
.await?;
let completed_job = RunJob::from(JobPayload::ScriptHub {
path: format!("hub/{version}/test/ok"),
apply_preprocessor: false,
})
.run_until_complete(&db, false, server.addr.port())
.await;
assert!(completed_job.success, "Hub script should have succeeded");
let mut handler_parent = None;
for _ in 0..50 {
handler_parent = sqlx::query_scalar::<_, Option<uuid::Uuid>>(
"SELECT parent_job FROM v2_job
WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'",
)
.fetch_optional(&db)
.await?;
if handler_parent.is_some() {
break;
}
tokio::time::sleep(tokio::time::Duration::from_millis(100)).await;
}
assert_eq!(
handler_parent,
Some(Some(completed_job.id)),
"Workspace success handler should handle the successful hub script"
);
Ok(())
}
backend/tests/suspend_resume.rs
+24 -24
View File
@@ -63,11 +63,11 @@ mod suspend_resume {
"op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "\
export async function main(n, port, op) {\
const job = Deno.env.get('WM_JOB_ID');
const token = Deno.env.get('WM_TOKEN');
const job = process.env.WM_JOB_ID;
const token = process.env.WM_TOKEN;
const r = await fetch(
`http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\
{\
@@ -102,7 +102,7 @@ mod suspend_resume {
"resumes": { "type": "javascript", "expr": "resumes", },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }"
},
"suspend": {
@@ -116,7 +116,7 @@ mod suspend_resume {
"resumes": { "type": "javascript", "expr": "resumes", },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(last, resume, resumes) { return { last, resume, resumes } }"
},
}],
@@ -256,11 +256,11 @@ mod suspend_resume {
"port": { "type": "javascript", "expr": "flow_input.port" },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "\
export async function main(port) {\
const job = Deno.env.get('WM_JOB_ID');\
const token = Deno.env.get('WM_TOKEN');\
const job = process.env.WM_JOB_ID;\
const token = process.env.WM_TOKEN;\
const secret = await (await fetch(\
`http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\
{ headers: { 'Authorization': `Bearer ${token}` } }\
@@ -278,7 +278,7 @@ mod suspend_resume {
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'b ran' }",
},
// The gate holds `b` back, so `b` never runs and its error policy describes
@@ -290,7 +290,7 @@ mod suspend_resume {
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'handled' }",
},
},
@@ -343,7 +343,7 @@ mod suspend_resume {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step1'; }"
},
"suspend": {
@@ -355,7 +355,7 @@ mod suspend_resume {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step2 - after approval'; }"
}
}]
@@ -454,7 +454,7 @@ mod suspend_resume {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step1'; }"
},
"suspend": {
@@ -466,7 +466,7 @@ mod suspend_resume {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step2 - after approval'; }"
}
}]
@@ -556,7 +556,7 @@ mod suspend_resume {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step1'; }"
},
"suspend": {
@@ -567,7 +567,7 @@ mod suspend_resume {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step2 - after approval'; }"
}
}]
@@ -658,7 +658,7 @@ mod suspend_resume {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step1'; }"
},
"suspend": {
@@ -670,7 +670,7 @@ mod suspend_resume {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step2 - after approval'; }"
}
}]
@@ -764,7 +764,7 @@ mod suspend_resume {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step1'; }"
},
"suspend": {
@@ -776,7 +776,7 @@ mod suspend_resume {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 'step2 - after approval'; }"
}
}]
@@ -933,7 +933,7 @@ mod suspend_resume {
"suspend": { "required_events": 1, "timeout": 86400 },
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export async function main() { return 1 }",
"input_transforms": {},
},
@@ -942,7 +942,7 @@ mod suspend_resume {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export async function main() { return 42 }",
"input_transforms": {},
},
@@ -980,7 +980,7 @@ mod suspend_resume {
"suspend": { "required_events": APPROVALS, "timeout": 86400 },
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export async function main() { return 1 }",
"input_transforms": {},
},
@@ -989,7 +989,7 @@ mod suspend_resume {
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(resumes) { return resumes.length }",
"input_transforms": {
"resumes": { "type": "javascript", "expr": "resumes" },
backend/tests/wac_suspend_started_at.rs
+88
View File
@@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool<Postgres>) -> anyhow::Res
Ok(())
}
/// Each park clears `started_at`, so the completion would otherwise store only the last
/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that
/// started an hour late. The completed row spans the workflow from its first start, and
/// keeps the last segment aside: that is what is metered and what worker-time readers sum.
#[sqlx::test]
async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment(
db: Pool<Postgres>,
) -> anyhow::Result<()> {
let job_id = Uuid::new_v4();
sqlx::query(
"INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \
permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \
VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \
'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)",
)
.bind(job_id)
.execute(&db)
.await?;
sqlx::query(
"INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \
VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')",
)
.bind(job_id)
.execute(&db)
.await?;
// Two parks, each followed by the pull re-stamping `started_at`: the second park must
// not replace the first start with its own segment's.
for resumed_ago in ["10 minutes", "3 seconds"] {
let mut tx = db.begin().await?;
suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?;
tx.commit().await?;
sqlx::query(
"UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \
WHERE id = $1",
)
.bind(job_id)
.bind(resumed_ago)
.execute(&db)
.await?;
}
let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db)
.await?
.unwrap();
let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?;
let (_, span) = windmill_queue::add_completed_job(
&db,
&job,
true,
false,
sqlx::types::Json(&result),
None,
0,
None,
false,
Some(3000),
false,
)
.await?;
let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option<i64>) = sqlx::query_as(
"SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \
(extras->>'wac_last_segment_ms')::bigint \
FROM v2_job_completed WHERE id = $1",
)
.bind(job_id)
.fetch_one(&db)
.await?;
assert!(
(started_ago_s - 3600.0).abs() < 60.0,
"started_at must be the first start, got {started_ago_s}s ago"
);
assert!(
(duration_ms - 3_600_000).abs() < 60_000,
"duration_ms must be the wall time since the first start, got {duration_ms}"
);
assert_eq!(last_segment_ms, Some(3000));
assert_eq!(
span.duration_ms, duration_ms,
"a flow records the step with the span the row holds"
);
assert!(span.outlasts_run(job.started_at));
assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter");
Ok(())
}
backend/tests/wm_token_confinement.rs
+1 -1
View File
@@ -319,7 +319,7 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool<Postgres>) -> anyho
);
}
// ...and the `settings/global` keys on the allowlist, which the CLI reads from a job: on a
// git-sync push, and in `u/admin/hub_sync`. `ws_base_url` is the control: the handler
// git-sync push, and in `wmill hub pull`. `ws_base_url` is the control: the handler
// leaves it as ungated as these, so only the allowlist stops it.
for key in ["automate_username_creation", "uid", "hub_base_url"] {
let resp = authed(
backend/tests/worker.rs
+153 -45
View File
@@ -166,7 +166,7 @@ async fn test_iteration_parallel(db: Pool<Postgres>) -> anyhow::Result<()> {
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
async fn test_bun_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
@@ -183,7 +183,7 @@ async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
id: "a".to_string(),
value: FlowModuleValue::RawScript {
input_transforms: Default::default(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: numbers.to_string(),
path: None,
lock: None,
@@ -232,7 +232,7 @@ async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
},
)]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: doubles.to_string(),
path: None,
lock: None,
@@ -296,7 +296,7 @@ async fn test_deno_flow(db: Pool<Postgres>) -> anyhow::Result<()> {
let port = server.addr.port();
for i in 0..50 {
println!("deno flow iteration: {}", i);
println!("bun flow iteration: {}", i);
let job = run_job_in_new_worker_until_complete(&db, false, job.clone(), port).await;
// println!("job: {:#?}", job.flow_status);
let result = job.json_result().unwrap();
@@ -346,13 +346,13 @@ async fn test_identity(db: Pool<Postgres>) -> anyhow::Result<()> {
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
async fn test_bun_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let write_file = r#"export async function main(loop: boolean, i: number, path: string) {
await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`);
await Bun.write(`./shared/${path}`, `${loop} ${i}`);
}"#
.to_string();
@@ -373,7 +373,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: write_file.clone(),
path: None,
lock: None,
@@ -432,7 +432,7 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: write_file,
path: None,
lock: None,
@@ -472,9 +472,9 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") },
)]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"export async function main(path: string, path2: string) {
return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`);
return await Bun.file(`./shared/${path}`).text() + "," + await Bun.file(`./shared/${path2}`).text();
}"#
.to_string(),
path: None,
@@ -546,9 +546,9 @@ async fn test_deno_flow_same_worker(db: Pool<Postgres>) -> anyhow::Result<()> {
),
]
.into(),
language: ScriptLang::Deno,
language: ScriptLang::Bun,
content: r#"export async function main(path: string, loops: string[], path2: string) {
return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`);
return await Bun.file(`./shared/${path}`).text() + "," + loops + "," + await Bun.file(`./shared/${path2}`).text();
}"#
.to_string(),
path: None,
@@ -618,7 +618,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool<Postgres>) -> anyhow::R
"id": "c",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return 1 }",
}
}]
@@ -630,7 +630,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool<Postgres>) -> anyhow::R
"id": "b",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return 42 }",
}
}
@@ -671,7 +671,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return 42 }",
}
},
@@ -686,7 +686,7 @@ async fn test_flow_result_by_id(db: Pool<Postgres>) -> anyhow::Result<()> {
"value": {
"input_transforms": {"v": {"type": "javascript", "expr": "results.a"}},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(v){ return v }",
}
@@ -3857,7 +3857,7 @@ fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value {
}
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(array, i){ array.push(i); return array }",
}
})
@@ -3868,7 +3868,7 @@ fn module_failure() -> serde_json::Value {
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ throw Error('failure') }",
}
})
@@ -3886,7 +3886,7 @@ async fn test_branchone_simple(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [1] }",
}
},
@@ -3924,7 +3924,7 @@ async fn test_branchone_with_cond(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [1] }",
}
},
@@ -3962,7 +3962,7 @@ async fn test_branchall_sequential(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [1] }",
}
},
@@ -4002,7 +4002,7 @@ async fn test_branchall_simple(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [1] }",
}
},
@@ -4050,7 +4050,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [1] }",
}
},
@@ -4086,7 +4086,7 @@ async fn test_branchall_skip_failure(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [1] }",
}
},
@@ -4132,7 +4132,7 @@ async fn test_branchone_nested(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [] }",
}
},
@@ -4191,7 +4191,7 @@ async fn test_branchall_nested(db: Pool<Postgres>) -> anyhow::Result<()> {
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(){ return [1] }",
}
},
@@ -4257,7 +4257,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(n, l) { if (n == 0) throw Error(JSON.stringify(l)); return { l: [...l, 0] } }",
},
}, {
@@ -4268,7 +4268,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(n, l) { if (n == 1) throw Error(JSON.stringify(l)); return { l: [...l, 1] } }",
},
}, {
@@ -4278,7 +4278,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
"n": { "type": "javascript", "expr": "flow_input.n", },
},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(n, l) { if (n == 2) throw Error(JSON.stringify(l)); return { l: [...l, 2] } }",
},
}],
@@ -4286,7 +4286,7 @@ async fn test_failure_module(db: Pool<Postgres>) -> anyhow::Result<()> {
"value": {
"input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } },
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(error) { return { 'from failure module': error } }",
}
},
@@ -4475,7 +4475,7 @@ fn hanging_step_value() -> serde_json::Value {
serde_json::json!({
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export async function main() { await new Promise((r) => setTimeout(r, 600000)); }",
})
}
@@ -4487,7 +4487,7 @@ fn marker_failure_module() -> serde_json::Value {
"value": {
"input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } },
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main(error) { return { handled_unrecoverable: true, error } }",
}
})
@@ -4585,7 +4585,7 @@ async fn test_unrecoverable_failure_on_continue_on_error_runs_failure_module(
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return { ran_b: true }; }",
},
},
@@ -4624,7 +4624,7 @@ async fn test_run_wait_result_early_return_with_failure_module(
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {},
"content": "export function main() { throw new Error('boom'); }",
},
@@ -4632,7 +4632,7 @@ async fn test_run_wait_result_early_return_with_failure_module(
"failure_module": {
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {},
"content": "export function main() { return { recovered: true } }",
},
@@ -4720,7 +4720,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { throw new Error('boom'); }",
},
}],
@@ -4734,7 +4734,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return { ran_b: true } }",
},
}],
@@ -4742,7 +4742,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
"value": {
"input_transforms": {},
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return { handled: true, recover: true } }",
},
},
@@ -4797,14 +4797,23 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
server.close().await.unwrap();
#[cfg(feature = "quickjs")]
assert!(!chaining_error.success, "an error no handler recovered must keep the flow failed");
assert!(
!chaining_error.success,
"an error no handler recovered must keep the flow failed"
);
assert!(stopped.success, "a recovered failure inside a loop should end the flow as a success");
assert!(
stopped.success,
"a recovered failure inside a loop should end the flow as a success"
);
assert!(
stopped.json_result().unwrap().get("ran_b").is_none(),
"the step after the loop must not run, as without recovery"
);
assert_eq!(stopped_handler_runs, 1, "the loop must stop at the first failed iteration");
assert_eq!(
stopped_handler_runs, 1,
"the loop must stop at the first failed iteration"
);
assert!(skipping.success);
assert_eq!(
@@ -4815,7 +4824,11 @@ async fn test_failure_module_recover_inside_loop(db: Pool<Postgres>) -> anyhow::
assert_eq!(skipping_handler_runs, 3);
assert!(
!parallel.json_result().unwrap().to_string().contains("ran_b"),
!parallel
.json_result()
.unwrap()
.to_string()
.contains("ran_b"),
"the parallel loop fails as without recovery, so the step after it must not run"
);
Ok(())
@@ -4987,6 +5000,101 @@ async fn test_flow_lock_all(db: Pool<Postgres>) -> anyhow::Result<()> {
Ok(())
}
/// A deployed flow runs a multi-step loop body as the flow node its deploy stored, which
/// carries its own copy of the flow's `preserve_step_tags`. A preview never loads that node.
#[sqlx::test(fixtures("base"))]
async fn test_saved_flow_preserves_nested_step_tags(db: Pool<Postgres>) -> anyhow::Result<()> {
use futures::StreamExt;
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let client = reqwest::Client::new();
let flow = json!({
"path": "g/all/saved_step_tags", "summary": "", "description": "", "tag": "python3",
"schema": { "type": "object", "properties": {}, "required": [] },
"value": {
"preserve_step_tags": true,
"modules": [{
"id": "loop",
"value": {
"type": "forloopflow", "skip_failures": false,
"iterator": { "type": "static", "value": [2, 3] },
"modules": [{
"id": "own_tag",
"value": {
"type": "rawscript", "language": "bun", "lock": "", "tag": "bun",
"content": "export function main(n: number) { return n * n; }",
"input_transforms": {
"n": { "type": "javascript", "expr": "flow_input.iter.value" }
}
}
}, {
"id": "untagged",
"value": {
"type": "rawscript", "language": "bun", "lock": "",
"content": "export function main(n: number) { return n + 1; }",
"input_transforms": {
"n": { "type": "javascript", "expr": "results.own_tag" }
}
}
}]
}
}]
}
});
let mut completed = listen_for_completed_jobs(&db).await;
client
.post(format!(
"http://localhost:{port}/api/w/test-workspace/flows/create"
))
.bearer_auth("SECRET_TOKEN")
.json(&flow)
.send()
.await?
.error_for_status()?;
let deployment = in_test_worker(&db, completed.next(), port).await.unwrap();
let deployed = completed_job(deployment, &db).await;
assert!(deployed.success, "{:?}", deployed.result);
let job_id = client
.post(format!(
"http://localhost:{port}/api/w/test-workspace/jobs/run/f/g/all/saved_step_tags"
))
.bearer_auth("SECRET_TOKEN")
.json(&json!({}))
.send()
.await?
.error_for_status()?
.text()
.await?
.parse::<Uuid>()?;
in_test_worker(&db, completed.find(&job_id), port).await;
let job = completed_job(job_id, &db).await;
assert!(job.success, "{:?}", job.result);
let tags = sqlx::query_as::<_, (String, String)>(
"WITH RECURSIVE descendants AS (
SELECT id FROM v2_job WHERE id = $1
UNION ALL
SELECT child.id FROM v2_job child JOIN descendants ON child.parent_job = descendants.id
)
SELECT DISTINCT flow_step_id, tag FROM v2_job
WHERE id IN (SELECT id FROM descendants) AND flow_step_id IN ('own_tag', 'untagged')
ORDER BY flow_step_id, tag",
)
.bind(job_id)
.fetch_all(&db)
.await?;
assert_eq!(
tags,
vec![
("own_tag".to_owned(), "bun".to_owned()),
("untagged".to_owned(), "python3".to_owned())
]
);
Ok(())
}
#[cfg(feature = "deno_core")]
#[sqlx::test(fixtures("base"))]
@@ -5687,7 +5795,7 @@ async fn test_job_labels(db: Pool<Postgres>) -> anyhow::Result<()> {
console.log(greet)
return { greet, wm_labels: ["yolo", "greet", "greet", world] };
}"#,
"language": "deno",
"language": "bun",
"input_transforms": {
"world": { "type": "javascript", "expr": "flow_input.world" }
}
@@ -5896,7 +6004,7 @@ async fn test_flow_substep_tag_availability_check(db: Pool<Postgres>) -> anyhow:
"id": "a",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return 42; }",
"tag": "restricted-tag",
},
@@ -5956,7 +6064,7 @@ async fn test_flow_substep_tag_checked_on_resolved_value(db: Pool<Postgres>) ->
"id": id,
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"content": "export function main() { return { lang: 'bun' } }",
"tag": tag,
},
backend/tests/zombie_flow_recovery.rs
+11 -11
View File
@@ -78,7 +78,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children(
"id": "inner",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
@@ -91,7 +91,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children(
"id": "after",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
@@ -210,7 +210,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool<Postgres>) -> anyhow::Res
"id": "inner",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
@@ -223,7 +223,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool<Postgres>) -> anyhow::Res
"id": "after",
"value": {
"type": "rawscript",
"language": "deno",
"language": "bun",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
@@ -331,7 +331,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse(
{
"id": "inner_first",
"value": {
"type": "rawscript", "language": "deno",
"type": "rawscript", "language": "bun",
"input_transforms": {},
"content": "export function main() { return 'first' }"
}
@@ -339,7 +339,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse(
{
"id": "inner_second",
"value": {
"type": "rawscript", "language": "deno",
"type": "rawscript", "language": "bun",
"input_transforms": {
"first": { "type": "javascript", "expr": "results.inner_first" }
},
@@ -353,7 +353,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse(
{
"id": "after",
"value": {
"type": "rawscript", "language": "deno",
"type": "rawscript", "language": "bun",
"input_transforms": {
"b": { "type": "javascript", "expr": "results.branch" }
},
@@ -466,7 +466,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step(
"modules": [{
"id": "inner",
"value": {
"type": "rawscript", "language": "deno",
"type": "rawscript", "language": "bun",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
@@ -478,7 +478,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step(
{
"id": "after",
"value": {
"type": "rawscript", "language": "deno",
"type": "rawscript", "language": "bun",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
@@ -578,7 +578,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool<Postgres>) -> anyhow::Re
"modules": [{
"id": "inner",
"value": {
"type": "rawscript", "language": "deno",
"type": "rawscript", "language": "bun",
"input_transforms": {
"v": { "type": "javascript", "expr": "flow_input.iter.value" }
},
@@ -590,7 +590,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool<Postgres>) -> anyhow::Re
{
"id": "after",
"value": {
"type": "rawscript", "language": "deno",
"type": "rawscript", "language": "bun",
"input_transforms": {
"loop_res": { "type": "javascript", "expr": "results.fanout" }
},
backend/windmill-ai/Cargo.toml
+1
View File
@@ -28,6 +28,7 @@ eventsource-stream.workspace = true
futures.workspace = true
http.workspace = true
mime_guess.workspace = true
rand.workspace = true
reqwest.workspace = true
serde.workspace = true
serde_json.workspace = true
backend/windmill-ai/src/ai_bedrock.rs
+23
View File
@@ -477,6 +477,29 @@ pub async fn assume_role_with_oidc_token(
// Error Formatting
// ============================================================================
/// The error for a Bedrock stream that failed after it opened, retrying the exceptions the
/// Vercel AI SDK retries (`amazon-bedrock-stream-error.ts`) and a broken connection. The
/// SDK's own retries cover the opening call alone.
pub fn bedrock_stream_error<R: std::fmt::Debug>(
error: &aws_sdk_bedrockruntime::error::SdkError<
aws_sdk_bedrockruntime::types::error::ConverseStreamOutputError,
R,
>,
) -> Error {
let message = format!("Bedrock stream error: {}", format_bedrock_error(error));
match error.as_service_error() {
Some(e)
if !(e.is_internal_server_exception()
|| e.is_model_stream_error_exception()
|| e.is_service_unavailable_exception()
|| e.is_throttling_exception()) =>
{
Error::AIError(message)
}
_ => crate::retry::transient_error(message, None),
}
}
/// Format AWS SDK errors with detailed information
pub fn format_bedrock_error<E, R>(error: &aws_sdk_bedrockruntime::error::SdkError<E, R>) -> String
where
backend/windmill-ai/src/ai_google.rs
+56
View File
@@ -326,6 +326,17 @@ pub struct GeminiSSEEvent {
pub candidates: Option<Vec<GeminiSSECandidate>>,
#[serde(rename = "usageMetadata")]
pub usage_metadata: Option<GeminiUsageMetadata>,
#[serde(default)]
pub error: Option<crate::retry::StreamErrorBody>,
#[serde(rename = "promptFeedback", default)]
pub prompt_feedback: Option<GeminiPromptFeedback>,
}
/// Set when Gemini refuses the prompt itself, in which case no candidate follows.
#[derive(Deserialize, Debug)]
pub struct GeminiPromptFeedback {
#[serde(rename = "blockReason", default)]
pub block_reason: Option<String>,
}
// ============================================================================
@@ -550,10 +561,40 @@ pub fn openai_tools_to_gemini(
}
}
/// Follows a Gemini stream to tell a complete turn from one cut short.
#[derive(Default)]
pub struct GeminiTurn {
ended: bool,
has_output: bool,
}
impl GeminiTurn {
pub fn observe(&mut self, event: &GeminiParsedEvent) {
// A function call arrives whole and may come without a `finishReason`, so it
// counts too, as in gemini-cli's stream validation.
self.ended |= event.finish_reason.is_some() || !event.tool_calls.is_empty();
self.has_output |= event.text.is_some() || event.reasoning.is_some();
}
/// Whether the turn's final event arrived.
pub fn ended(&self) -> bool {
self.ended
}
/// Whether a stream that closed cleanly here holds the whole turn. Gemini 2.5 closes
/// its stream without a single event when thinking spends the whole
/// `maxOutputTokens`: an empty answer, which a retry would only get again, rather
/// than a cut one.
pub fn complete_at_close(&self) -> bool {
self.ended || !self.has_output
}
}
/// Parse one Gemini SSE data line into a [`GeminiParsedEvent`].
///
/// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`).
/// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating.
/// An error the stream reports is returned as one.
pub fn parse_gemini_sse_event(data: &str) -> Result<Option<GeminiParsedEvent>, Error> {
if data.is_empty() || data == "[DONE]" {
return Ok(None);
@@ -567,6 +608,21 @@ pub fn parse_gemini_sse_event(data: &str) -> Result<Option<GeminiParsedEvent>, E
}
};
if let Some(error) = event.error {
return Err(error.into_error("Gemini"));
}
// `BLOCK_REASON_UNSPECIFIED` is the protobuf enum's placeholder, not a block.
if let Some(reason) = event
.prompt_feedback
.and_then(|f| f.block_reason)
.filter(|r| !matches!(r.as_str(), "" | "BLOCK_REASON_UNSPECIFIED"))
{
return Err(Error::AIError(format!(
"Gemini blocked the prompt: {reason}"
)));
}
let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() };
let Some(candidates) = event.candidates else {
backend/windmill-ai/src/lib.rs
+1
View File
@@ -11,6 +11,7 @@ pub mod model_context;
pub mod providers;
pub mod proxy;
pub mod query_builder;
pub mod retry;
pub mod sse;
pub mod types;
pub mod utils;
backend/windmill-ai/src/providers/bedrock.rs
+36 -19
View File
@@ -9,11 +9,12 @@
use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL};
use crate::{
ai_bedrock::{
bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop,
bedrock_stream_event_to_reasoning_delta, bedrock_stream_event_to_text,
bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_delta_with_block_index,
bedrock_stream_event_to_tool_start, bedrock_stream_event_to_tool_start_with_block_index,
build_tool_config, create_inference_config, format_bedrock_error, json_to_document,
bedrock_model_supports_prompt_caching, bedrock_stream_error,
bedrock_stream_event_is_block_stop, bedrock_stream_event_to_reasoning_delta,
bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta,
bedrock_stream_event_to_tool_delta_with_block_index, bedrock_stream_event_to_tool_start,
bedrock_stream_event_to_tool_start_with_block_index, build_tool_config,
create_inference_config, format_bedrock_error, json_to_document,
openai_messages_to_bedrock, streaming_tool_calls_to_openai, BearerTokenProvider,
BedrockClient, StreamingToolCall,
},
@@ -24,6 +25,7 @@ use crate::{
image_handler::prepare_messages_for_api,
proxy::ProxyBuildArgs,
query_builder::{ParsedResponse, StreamEventSink},
retry::{openai_stream_error_chunk, truncated_stream_error},
types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef},
};
use bytes::Bytes;
@@ -461,26 +463,31 @@ pub fn sdk_stream_to_sse(
async_stream::stream! {
let mut stream = stream;
let mut state = BedrockSseStreamState::new(id, model, created);
let mut message_stopped = false;
loop {
let failure = loop {
match stream.recv().await {
Ok(Some(event)) => {
if matches!(
event,
aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_)
) {
message_stopped = true;
}
for chunk in bedrock_sse_chunks_for_event(&event, &mut state) {
yield Ok(chunk);
}
}
Ok(None) => break,
Err(e) => {
yield Err(std::io::Error::new(
std::io::ErrorKind::Other,
e.to_string(),
));
break;
}
Ok(None) => break (!message_stopped).then(truncated_stream_error),
Err(_) if message_stopped => break None,
Err(e) => break Some(bedrock_stream_error(&e)),
}
}
};
yield Ok(Bytes::from("data: [DONE]\n\n"));
match failure {
Some(error) => yield Ok(Bytes::from(openai_stream_error_chunk(&error))),
None => yield Ok(Bytes::from("data: [DONE]\n\n")),
}
}
}
@@ -1081,11 +1088,19 @@ impl BedrockQueryBuilder {
// Claude reasoning block for the turn (only populated when thinking is on),
// attached to the first tool call for replay before toolUse.
let mut reasoning: Option<crate::ai_types::BedrockExtraContent> = None;
let mut message_stopped = false;
// Process stream events using shared parsing functions
loop {
match stream.recv().await {
Ok(Some(event)) => {
if matches!(
event,
aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_)
) {
message_stopped = true;
}
// Fold reasoning deltas into the turn's reasoning block (for
// replay before toolUse, required by Claude when thinking is
// on) and stream the readable summary as a thinking affordance.
@@ -1161,11 +1176,13 @@ impl BedrockQueryBuilder {
}
}
Ok(None) => break,
Err(e) => {
return Err(Error::internal_err(format!("Bedrock stream error: {}", e)));
}
Err(_) if message_stopped => break,
Err(e) => return Err(bedrock_stream_error(&e)),
}
}
if !message_stopped {
return Err(truncated_stream_error());
}
// Send tool call events to stream processor
if let Some(processor) = stream_event_sink.as_ref() {
backend/windmill-ai/src/providers/google_ai.rs
+28 -25
View File
@@ -5,17 +5,20 @@ use crate::{
parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google,
GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiImageContent, GeminiImageRequest,
GeminiImageResponse, GeminiInlineData, GeminiPart, GeminiPredictContent, GeminiTextRequest,
GeminiThinkingConfig, GeminiTool,
GeminiThinkingConfig, GeminiTool, GeminiTurn,
},
image_handler::{download_and_encode_s3_image, prepare_messages_for_api},
proxy::{ProxyBuildArgs, ProxyRequest},
query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink},
retry::{
openai_stream_error_chunk, send_with_retries, transient_error, truncated_stream_error,
},
sse::{GeminiSSEParser, SSEParser},
types::*,
};
use async_trait::async_trait;
use bytes::Bytes;
use eventsource_stream::Eventsource;
use eventsource_stream::{EventStreamError, Eventsource};
use futures::{stream::BoxStream, StreamExt};
use http::{header, HeaderMap, HeaderValue, Method, StatusCode};
use serde::Deserialize;
@@ -336,9 +339,7 @@ pub async fn handle_google_ai_chat_proxy(
) -> Result<GoogleAIProxyResponse, Error> {
let GoogleAIProxyRequest { request, model, stream } = build_google_ai_chat_proxy_request(args)?;
let response =
send_google_ai_proxy_request(client, request, "Failed to send request to Gemini API")
.await?;
let response = send_google_ai_proxy_request(client, request).await?;
if stream {
Ok(convert_streaming_response(response, &model))
@@ -356,8 +357,7 @@ pub async fn handle_google_ai_models_proxy(
args: &ProxyBuildArgs<'_>,
) -> Result<GoogleAIProxyResponse, Error> {
let request = build_google_ai_models_proxy_request(args);
let response =
send_google_ai_proxy_request(client, request, "Failed to fetch Gemini models").await?;
let response = send_google_ai_proxy_request(client, request).await?;
let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| {
Error::internal_err(format!("Failed to parse Gemini models response: {}", e))
@@ -505,26 +505,13 @@ fn add_google_ai_auth_header(headers: &mut Vec<(String, String)>, api_key: &str,
async fn send_google_ai_proxy_request(
client: &reqwest::Client,
proxy_request: ProxyRequest,
send_error_message: &str,
) -> Result<reqwest::Response, Error> {
let mut request = client.request(proxy_request.method.clone(), &proxy_request.url);
for (header_name, header_value) in &proxy_request.headers {
request = request.header(header_name.as_str(), header_value.as_str());
}
let response = request
.body(proxy_request.body)
.send()
.await
.map_err(|e| Error::internal_err(format!("{}: {}", send_error_message, e)))?;
if let Err(e) = response.error_for_status_ref() {
let status = e.status().map(|s| s.to_string()).unwrap_or_default();
let body = response.text().await.unwrap_or_default();
return Err(Error::AIError(format!("{}: {}", status, body)));
}
Ok(response)
send_with_retries(request.body(proxy_request.body)).await
}
fn convert_streaming_response(response: reqwest::Response, model: &str) -> GoogleAIProxyResponse {
@@ -535,10 +522,15 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl
let openai_sse_stream = async_stream::stream! {
tokio::pin!(gemini_sse_stream);
let mut tool_call_index: usize = 0;
while let Some(event) = gemini_sse_stream.next().await {
let mut turn = GeminiTurn::default();
let failure = loop {
let Some(event) = gemini_sse_stream.next().await else {
break (!turn.complete_at_close()).then(truncated_stream_error);
};
match event {
Ok(event) => match parse_gemini_sse_event(&event.data) {
Ok(Some(parsed)) => {
turn.observe(&parsed);
for chunk in gemini_event_to_openai_sse_chunks(
&parsed, &id, &model, &mut tool_call_index,
) {
@@ -546,12 +538,23 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl
}
}
Ok(None) => {}
Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e),
Err(e) => break Some(e),
},
Err(EventStreamError::Transport(_)) if turn.ended() => break None,
Err(EventStreamError::Transport(e)) => {
break Some(transient_error(
format!("The connection to Gemini broke off mid-response: {e}"),
None,
));
}
Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e),
}
}
yield Ok::<Bytes, reqwest::Error>(Bytes::from("data: [DONE]\n\n"));
};
let last = match failure {
Some(error) => openai_stream_error_chunk(&error),
None => "data: [DONE]\n\n".to_string(),
};
yield Ok::<Bytes, reqwest::Error>(Bytes::from(last));
}
.boxed();
backend/windmill-ai/src/providers/other.rs
+1
View File
@@ -266,6 +266,7 @@ impl QueryBuilder for OtherQueryBuilder {
mut events_str,
stream_event_processor,
usage: openai_usage,
..
} = openai_sse_parser;
// Process streaming events with error handling
backend/windmill-ai/src/retry.rs
+474
View File
@@ -0,0 +1,474 @@
use std::time::Duration;
use reqwest::{header::HeaderMap, RequestBuilder, Response, StatusCode};
use windmill_common::error::{to_anyhow, Error};
/// How many times a provider call is retried after its first attempt.
pub const MAX_RETRIES: u32 = 3;
const BASE_DELAY: Duration = Duration::from_secs(1);
const MAX_BACKOFF: Duration = Duration::from_secs(10);
/// A provider that asks for a longer wait is out of capacity for longer than anyone should
/// block on (a spent daily quota, say), so its error is returned instead of waited out.
const MAX_RETRY_AFTER: Duration = Duration::from_secs(60);
/// A provider call that failed for a reason a later attempt may not meet: the provider was
/// overloaded, throttled or unreachable, or its response broke off part way.
#[derive(Debug)]
pub struct TransientAIError {
pub message: String,
/// The wait the provider asked for, when it named one.
pub retry_after: Option<Duration>,
}
impl std::fmt::Display for TransientAIError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for TransientAIError {}
pub fn transient_error(message: impl Into<String>, retry_after: Option<Duration>) -> Error {
to_anyhow(TransientAIError { message: message.into(), retry_after }).into()
}
pub fn as_transient(error: &Error) -> Option<&TransientAIError> {
match error {
Error::Anyhow { error, .. } => error.downcast_ref::<TransientAIError>(),
_ => None,
}
}
/// The error to report once a transient failure is no longer retried, without the source
/// location `Error::Anyhow` renders.
pub fn into_final_error(error: Error) -> Error {
match as_transient(&error) {
Some(transient) => Error::AIError(transient.message.clone()),
None => error,
}
}
/// The statuses the OpenAI and Anthropic SDKs and the Vercel AI SDK retry.
fn is_retryable_status(status: u16) -> bool {
matches!(status, 408 | 409 | 429 | 500..=599)
}
/// The error for a provider's error response. The rules follow the OpenAI and Anthropic
/// SDKs (`x-should-retry`, then the status) and gemini-cli (a daily quota is spent until
/// tomorrow, and the wait Gemini asks for is in its body).
pub fn response_error(
status: StatusCode,
headers: &HeaderMap,
body: &str,
message: String,
) -> Error {
let retryable = match headers.get("x-should-retry").and_then(|v| v.to_str().ok()) {
Some("true") => true,
Some("false") => false,
_ => {
is_retryable_status(status.as_u16())
// OpenAI answers a spent balance with the 429 it also uses for rate limits.
&& !body.contains("insufficient_quota")
&& !google_quota_spent_for_the_day(body)
}
};
if retryable {
transient_error(
message,
retry_after(headers).or_else(|| google_retry_delay(body)),
)
} else {
Error::AIError(message)
}
}
/// The `details` of a Google API error body, where Gemini puts its quota and retry hints.
fn google_error_details(body: &str) -> Vec<serde_json::Value> {
serde_json::from_str::<serde_json::Value>(body)
.ok()
.and_then(|v| v.get("error")?.get("details")?.as_array().cloned())
.unwrap_or_default()
}
fn google_quota_spent_for_the_day(body: &str) -> bool {
google_error_details(body).iter().any(|detail| {
detail
.get("violations")
.and_then(|v| v.as_array())
.is_some_and(|violations| {
violations.iter().any(|violation| {
violation
.get("quotaId")
.and_then(|id| id.as_str())
.is_some_and(|id| id.contains("PerDay") || id.contains("Daily"))
})
})
})
}
/// The `RetryInfo.retryDelay` of a Google API error, a protobuf duration such as `"53s"`.
fn google_retry_delay(body: &str) -> Option<Duration> {
google_error_details(body).iter().find_map(|detail| {
let delay = detail.get("retryDelay")?.as_str()?.strip_suffix('s')?;
let seconds = delay
.parse::<f64>()
.ok()
.filter(|v| v.is_finite() && *v >= 0.0)?;
Some(Duration::try_from_secs_f64(seconds).unwrap_or(Duration::MAX))
})
}
/// The wait a provider asked for in its `retry-after-ms` or `retry-after` header. An
/// HTTP-date `retry-after` is ignored, which falls back to the computed backoff.
fn retry_after(headers: &HeaderMap) -> Option<Duration> {
let seconds = |name: &str, scale: f64| {
headers
.get(name)?
.to_str()
.ok()?
.trim()
.parse::<f64>()
.ok()
.filter(|v| v.is_finite() && *v >= 0.0)
.map(|v| Duration::try_from_secs_f64(v / scale).unwrap_or(Duration::MAX))
};
seconds("retry-after-ms", 1000.0).or_else(|| seconds("retry-after", 1.0))
}
/// The error object a provider sends inside a stream: OpenAI-compatible chunks carry
/// `type` and a string or status `code`, Anthropic `type`, Gemini a status `code` and
/// `status`.
#[derive(Debug, Default, serde::Deserialize)]
pub struct StreamErrorBody {
#[serde(default)]
pub message: Option<String>,
#[serde(default, rename = "type")]
pub kind: Option<String>,
#[serde(default)]
pub code: Option<serde_json::Value>,
#[serde(default)]
pub status: Option<String>,
}
impl StreamErrorBody {
/// The error for a failure an OpenAI-compatible or Gemini stream reported after opening
/// with a 200, classified as the Vercel AI SDK classifies OpenAI stream errors
/// (`openai-stream-error.ts`): the status `code` holds, else the status its code and type
/// name, else a server error.
pub fn into_error(self, provider: &str) -> Error {
let code = self.code_str();
let discriminator = [code.as_deref(), self.kind.as_deref()]
.into_iter()
.flatten()
.collect::<Vec<_>>()
.join(" ")
.to_lowercase();
let has = |terms: &[&str]| terms.iter().any(|term| discriminator.contains(term));
let status = match code.as_deref().and_then(|code| code.parse::<u16>().ok()) {
Some(status @ 400..=599) => status,
_ if has(&["insufficient_quota", "rate_limit"]) => 429,
_ if has(&["authentication"]) => 401,
_ if has(&["permission"]) => 403,
_ if has(&["not_found"]) => 404,
_ if has(&["invalid", "bad_request", "context_length"]) => 400,
_ if has(&["overload"]) => 503,
_ if has(&["timeout"]) => 504,
_ => 500,
};
let retryable = !has(&["insufficient_quota"]) && is_retryable_status(status);
self.into_classified_error(provider, retryable)
}
/// The error for a failure an Anthropic stream reported after opening with a 200. The
/// kinds retried are those the Vercel AI SDK retries, plus `timeout_error`, which
/// Anthropic documents as its 504; any other kind is a fault in the request or account.
pub fn into_anthropic_error(self) -> Error {
let retryable = matches!(
self.kind.as_deref(),
Some("api_error" | "overloaded_error" | "rate_limit_error" | "timeout_error")
);
self.into_classified_error("Anthropic", retryable)
}
fn code_str(&self) -> Option<String> {
match self.code.as_ref()? {
serde_json::Value::String(code) => Some(code.clone()),
serde_json::Value::Number(code) => Some(code.to_string()),
_ => None,
}
}
fn into_classified_error(self, provider: &str, retryable: bool) -> Error {
let kind = [self.kind.clone(), self.code_str(), self.status]
.into_iter()
.flatten()
.collect::<Vec<_>>()
.join("/");
let message = format!(
"{provider} stream error{}: {}",
(!kind.is_empty())
.then(|| format!(" ({kind})"))
.unwrap_or_default(),
self.message.as_deref().unwrap_or("no details")
);
if retryable {
transient_error(message, None)
} else {
Error::AIError(message)
}
}
}
/// The error for a stream that closed before the provider sent the event ending a
/// complete response: cut off by the provider or by a proxy in between.
pub fn truncated_stream_error() -> Error {
transient_error(
"The AI provider's response stream ended before the response was complete",
None,
)
}
/// An OpenAI-format stream chunk reporting `error`, which ends a response converted to that
/// format after it started streaming: the chat's OpenAI SDK throws on a chunk carrying
/// `error`, where a bare end of stream would read as a finished answer.
pub fn openai_stream_error_chunk(error: &Error) -> String {
let message = as_transient(error).map_or_else(|| error.to_string(), |t| t.message.clone());
format!(
"data: {}\n\n",
serde_json::json!({ "error": { "message": message } })
)
}
/// Sends a provider request, retrying a transient failure while nothing of the response
/// has been read: a failed connection or an error status. Once retries are spent the error
/// carries the provider's answer, as a single attempt's would.
pub async fn send_with_retries(request: RequestBuilder) -> Result<Response, Error> {
let mut backoff = Backoff::default();
loop {
// A streamed body cannot be replayed, so such a request is sent once.
let Some(attempt) = request.try_clone() else {
return send_once(request).await.map_err(into_final_error);
};
let error = match send_once(attempt).await {
Ok(response) => return Ok(response),
Err(error) => error,
};
let Some(delay) = as_transient(&error).and_then(|t| backoff.next_delay(t)) else {
return Err(into_final_error(error));
};
tracing::warn!(
"AI provider call failed ({error}); retrying in {:.1}s ({}/{MAX_RETRIES})",
delay.as_secs_f64(),
backoff.retries()
);
tokio::time::sleep(delay).await;
}
}
/// The error for a provider request that failed before any response arrived. The request
/// timeout is the caller's whole budget, so a timed-out call leaves no time for another
/// attempt; a connect timeout has its own, shorter one.
pub fn send_error(e: reqwest::Error, context: &str) -> Error {
let message = format!("{context}: {e}");
if e.is_builder() || (e.is_timeout() && !e.is_connect()) {
Error::AIError(message)
} else {
transient_error(message, None)
}
}
async fn send_once(request: RequestBuilder) -> Result<Response, Error> {
let response = request
.send()
.await
.map_err(|e| send_error(e, "Failed to reach the AI provider"))?;
let status = response.status();
if status.is_client_error() || status.is_server_error() {
let headers = response.headers().clone();
let body = response.text().await.unwrap_or_default();
return Err(response_error(status, &headers, &body, body.clone()));
}
Ok(response)
}
/// Counts the retries of one provider call and spaces them out.
#[derive(Debug, Default)]
pub struct Backoff {
retries: u32,
}
impl Backoff {
/// The wait before retrying a call that failed with `error`, or `None` once it should
/// fail instead.
pub fn next_delay(&mut self, error: &TransientAIError) -> Option<Duration> {
if self.retries >= MAX_RETRIES {
return None;
}
let delay = match error.retry_after {
Some(wait) if wait > MAX_RETRY_AFTER => return None,
// A zero wait would retry back to back, so it gets the computed backoff.
Some(wait) if !wait.is_zero() => wait,
// Jittered, so callers throttled together do not come back together.
_ => (BASE_DELAY * 2u32.pow(self.retries))
.min(MAX_BACKOFF)
.mul_f64(0.5 + rand::random::<f64>() * 0.5),
};
self.retries += 1;
Some(delay)
}
/// Retries granted so far.
pub fn retries(&self) -> u32 {
self.retries
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Error responses as providers send them, each with the retry decision of the client
/// the rule is taken from.
#[test]
fn classifies_error_responses() {
const GEMINI_QUOTA: &str = r#"{"error":{"code":429,"status":"RESOURCE_EXHAUSTED","details":[{"@type":"type.googleapis.com/google.rpc.QuotaFailure","violations":[{"quotaId":"QUOTA_ID"}]},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"53s"}]}}"#;
let cases: &[(u16, &[(&str, &str)], &str, Option<Option<u64>>)] = &[
(
429,
&[],
r#"{"error":{"code":"rate_limit_exceeded"}}"#,
Some(None),
),
(
429,
&[],
r#"{"error":{"type":"insufficient_quota","code":"insufficient_quota"}}"#,
None,
),
(
529,
&[],
r#"{"type":"error","error":{"type":"overloaded_error"}}"#,
Some(None),
),
(501, &[], "", Some(None)),
(400, &[], "", None),
(401, &[], "", None),
(400, &[("x-should-retry", "true")], "", Some(None)),
(503, &[("x-should-retry", "false")], "", None),
(429, &[("retry-after", "2")], "", Some(Some(2))),
(
429,
&[],
&GEMINI_QUOTA.replace("QUOTA_ID", "GenerateRequestsPerMinutePerProjectPerModel"),
Some(Some(53)),
),
(
429,
&[],
&GEMINI_QUOTA.replace(
"QUOTA_ID",
"GenerateRequestsPerDayPerProjectPerModel-FreeTier",
),
None,
),
];
for (status, headers, body, expected) in cases {
let headers = headers
.iter()
.map(|(k, v)| (k.parse().unwrap(), v.parse().unwrap()))
.collect::<HeaderMap>();
let error = response_error(
StatusCode::from_u16(*status).unwrap(),
&headers,
body,
String::new(),
);
let got = as_transient(&error).map(|t| t.retry_after.map(|d| d.as_secs()));
assert_eq!(&got, expected, "{status} {headers:?} {body}");
}
}
/// In-stream error objects as providers send them, after the 200 opened the stream.
#[test]
fn classifies_stream_errors() {
let retried = |json: &str, anthropic: bool| {
let body: StreamErrorBody = serde_json::from_str(json).unwrap();
let error = if anthropic {
body.into_anthropic_error()
} else {
body.into_error("AI provider")
};
as_transient(&error).is_some()
};
for (json, expected) in [
(r#"{"message":"m","type":"server_error"}"#, true),
(
r#"{"message":"m","type":"requests","code":"rate_limit_exceeded"}"#,
true,
),
(
r#"{"message":"m","type":"insufficient_quota","code":"insufficient_quota"}"#,
false,
),
(
r#"{"message":"m","type":"invalid_request_error","code":"context_length_exceeded"}"#,
false,
),
// LiteLLM's proxy puts the upstream status in `code` as a string.
(r#"{"message":"m","type":"None","code":"400"}"#, false),
(r#"{"message":"m","code":"503"}"#, true),
(r#"{"code":503,"message":"m","status":"UNAVAILABLE"}"#, true),
(
r#"{"code":400,"message":"m","status":"INVALID_ARGUMENT"}"#,
false,
),
(r#"{"message":"m"}"#, true),
] {
assert_eq!(retried(json, false), expected, "{json}");
}
for (kind, expected) in [
("overloaded_error", true),
("api_error", true),
("rate_limit_error", true),
("timeout_error", true),
("billing_error", false),
("invalid_request_error", false),
("some_new_error", false),
] {
let json = format!(r#"{{"type":"{kind}","message":"m"}}"#);
assert_eq!(retried(&json, true), expected, "{kind}");
}
}
#[test]
fn honours_retry_after_up_to_a_bound() {
let mut headers = HeaderMap::new();
headers.insert("retry-after", "2".parse().unwrap());
assert_eq!(retry_after(&headers), Some(Duration::from_secs(2)));
headers.insert("retry-after-ms", "1500".parse().unwrap());
assert_eq!(retry_after(&headers), Some(Duration::from_millis(1500)));
headers.remove("retry-after-ms");
headers.insert("retry-after", "1e30".parse().unwrap());
assert_eq!(retry_after(&headers), Some(Duration::MAX));
let mut backoff = Backoff::default();
let asked = |secs| TransientAIError {
message: String::new(),
retry_after: Some(Duration::from_secs(secs)),
};
assert_eq!(backoff.next_delay(&asked(5)), Some(Duration::from_secs(5)));
assert_eq!(backoff.next_delay(&asked(3600)), None);
}
#[test]
fn stops_after_max_retries() {
let mut backoff = Backoff::default();
let error = TransientAIError { message: String::new(), retry_after: None };
for _ in 0..MAX_RETRIES {
let delay = backoff.next_delay(&error).expect("retry");
assert!(delay <= MAX_BACKOFF);
}
assert_eq!(backoff.next_delay(&error), None);
}
}
backend/windmill-ai/src/sse.rs
+231 -13
View File
@@ -1,6 +1,6 @@
use std::collections::HashMap;
use eventsource_stream::Eventsource;
use eventsource_stream::{EventStreamError, Eventsource};
use indexmap::IndexMap;
use reqwest::Response;
use serde::Deserialize;
@@ -8,12 +8,13 @@ use tokio_stream::StreamExt;
use windmill_common::{error::Error, utils::rd_string};
use crate::{
ai_google::{parse_gemini_sse_event, GeminiUsageMetadata},
ai_google::{parse_gemini_sse_event, GeminiTurn, GeminiUsageMetadata},
ai_types::UrlCitation,
ai_types::{
AnthropicExtraContent, ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall,
},
query_builder::StreamEventSink,
retry::{transient_error, truncated_stream_error, StreamErrorBody},
types::{StreamingEvent, TokenUsage},
};
@@ -43,6 +44,8 @@ pub struct OpenAIChoiceDelta {
#[derive(Deserialize)]
pub struct OpenAIChoice {
pub delta: Option<OpenAIChoiceDelta>,
#[serde(default)]
pub finish_reason: Option<String>,
}
/// Nested prompt token details returned by the Chat Completions API.
@@ -87,6 +90,8 @@ pub struct OpenAISSEEvent {
pub choices: Option<Vec<OpenAIChoice>>,
#[serde(default)]
pub usage: Option<OpenAIChatUsage>,
#[serde(default)]
pub error: Option<StreamErrorBody>,
}
lazy_static::lazy_static! {
@@ -99,6 +104,14 @@ lazy_static::lazy_static! {
pub trait SSEParser {
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>;
/// Whether the stream delivered the event that ends a complete response.
fn is_complete(&self) -> bool;
/// Whether a stream that closed cleanly holds a complete response.
fn is_complete_at_close(&self) -> bool {
self.is_complete()
}
async fn parse_events(&mut self, response: Response) -> Result<(), Error> {
let mut stream = response.bytes_stream().eventsource();
let mut consecutive_errors = 0;
@@ -113,6 +126,18 @@ pub trait SSEParser {
self.parse_event_data(&event.data).await?;
}
// The body ends right after a transport error, so it cannot be skipped
// like a malformed event. A timeout spent the caller's whole budget.
Err(EventStreamError::Transport(_)) if self.is_complete() => break,
Err(EventStreamError::Transport(e)) => {
let message =
format!("The connection to the AI provider broke off mid-response: {e}");
return Err(if e.is_timeout() {
Error::AIError(message)
} else {
transient_error(message, None)
});
}
Err(e) => {
consecutive_errors += 1;
tracing::error!("Failed to parse SSE event: {}", e);
@@ -130,6 +155,9 @@ pub trait SSEParser {
}
}
if !self.is_complete_at_close() {
return Err(truncated_stream_error());
}
Ok(())
}
}
@@ -145,6 +173,9 @@ pub struct OpenAISSEParser {
pub stream_event_processor: Box<dyn StreamEventSink>,
/// Token usage from final chunk (when stream_options.include_usage is true)
pub usage: Option<OpenAIChatUsage>,
/// Set by `[DONE]` or by a choice's `finish_reason`: OpenAI-compatible gateways do not
/// all send `[DONE]`.
complete: bool,
}
impl OpenAISSEParser {
@@ -156,13 +187,19 @@ impl OpenAISSEParser {
events_str: String::new(),
stream_event_processor,
usage: None,
complete: false,
}
}
}
impl SSEParser for OpenAISSEParser {
fn is_complete(&self) -> bool {
self.complete
}
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
if data == "[DONE]" {
self.complete = true;
return Ok(());
}
@@ -173,13 +210,21 @@ impl SSEParser for OpenAISSEParser {
.ok();
if let Some(event) = event {
if let Some(error) = event.error {
return Err(error.into_error("AI provider"));
}
// Extract usage from final chunk (when stream_options.include_usage is true)
if let Some(usage) = event.usage {
self.usage = Some(usage);
}
if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) {
if let Some(delta) = choices.remove(0).delta {
let choice = choices.remove(0);
if choice.finish_reason.is_some() {
self.complete = true;
}
if let Some(delta) = choice.delta {
if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) {
self.accumulated_reasoning.push_str(&reasoning);
let event = StreamingEvent::ReasoningTokenDelta { content: reasoning };
@@ -345,6 +390,8 @@ pub enum AnthropicSSEEvent {
MessageDelta {
#[serde(default)]
usage: Option<AnthropicUsage>,
#[serde(default)]
delta: Option<AnthropicMessageDelta>,
},
#[serde(rename = "message_stop")]
MessageStop {},
@@ -353,12 +400,18 @@ pub enum AnthropicSSEEvent {
#[serde(rename = "error")]
Error {
#[serde(default)]
message: Option<String>,
error: StreamErrorBody,
},
#[serde(other)]
Unknown,
}
#[derive(Deserialize, Debug)]
pub struct AnthropicMessageDelta {
#[serde(default)]
pub stop_reason: Option<String>,
}
/// Tracks state of a content block during streaming
#[derive(Debug)]
#[allow(dead_code)]
@@ -391,6 +444,9 @@ pub struct AnthropicSSEParser {
/// can be replayed before `tool_use` (required by Claude when thinking is on).
pending_reasoning: Option<AnthropicExtraContent>,
reasoning_attached: bool,
/// Set by `message_stop` or by the `stop_reason` of the `message_delta` sent just
/// before it, so a gateway that drops `message_stop` does not fail a whole answer.
complete: bool,
}
impl AnthropicSSEParser {
@@ -407,11 +463,16 @@ impl AnthropicSSEParser {
usage: None,
pending_reasoning: None,
reasoning_attached: false,
complete: false,
}
}
}
impl SSEParser for AnthropicSSEParser {
fn is_complete(&self) -> bool {
self.complete
}
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
let event: Option<AnthropicSSEEvent> = serde_json::from_str(data)
.inspect_err(|e| {
@@ -583,9 +644,8 @@ impl SSEParser for AnthropicSSEParser {
.await?;
}
}
AnthropicSSEEvent::Error { message } => {
let error_msg = message.unwrap_or_else(|| "Unknown error".to_string());
tracing::error!("Anthropic streaming error: {}", error_msg);
AnthropicSSEEvent::Error { error } => {
return Err(error.into_anthropic_error());
}
AnthropicSSEEvent::MessageStart { message } => {
// The only event carrying the prompt-side counts. `message_delta`
@@ -595,7 +655,10 @@ impl SSEParser for AnthropicSSEParser {
self.usage = Some(usage);
}
}
AnthropicSSEEvent::MessageDelta { usage } => {
AnthropicSSEEvent::MessageDelta { usage, delta } => {
if delta.is_some_and(|d| d.stop_reason.is_some()) {
self.complete = true;
}
if let Some(u) = usage {
match &mut self.usage {
// Field by field, so the prompt counts from `message_start`
@@ -614,10 +677,9 @@ impl SSEParser for AnthropicSSEParser {
}
}
}
AnthropicSSEEvent::MessageStop {} => self.complete = true,
// Ignore other events
AnthropicSSEEvent::MessageStop {}
| AnthropicSSEEvent::Ping {}
| AnthropicSSEEvent::Unknown => {}
AnthropicSSEEvent::Ping {} | AnthropicSSEEvent::Unknown => {}
}
}
@@ -645,6 +707,7 @@ pub struct GeminiSSEParser {
pub annotations: Vec<UrlCitation>,
pub used_websearch: bool,
pub usage: Option<GeminiUsageMetadata>,
turn: GeminiTurn,
}
impl GeminiSSEParser {
@@ -659,15 +722,25 @@ impl GeminiSSEParser {
annotations: Vec::new(),
used_websearch: false,
usage: None,
turn: GeminiTurn::default(),
}
}
}
impl SSEParser for GeminiSSEParser {
fn is_complete(&self) -> bool {
self.turn.ended()
}
fn is_complete_at_close(&self) -> bool {
self.turn.complete_at_close()
}
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
let Some(parsed) = parse_gemini_sse_event(data)? else {
return Ok(());
};
self.turn.observe(&parsed);
if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) {
self.accumulated_reasoning.push_str(&reasoning);
@@ -797,6 +870,8 @@ impl OpenAIResponsesUsage {
pub struct OpenAIResponsesResponse {
#[serde(default)]
pub usage: Option<OpenAIResponsesUsage>,
#[serde(default)]
pub error: Option<StreamErrorBody>,
}
/// SSE event types for OpenAI Responses API streaming
@@ -833,6 +908,20 @@ pub enum OpenAIResponsesSSEEvent {
#[serde(rename = "response.incomplete")]
Incomplete { response: OpenAIResponsesResponse },
#[serde(rename = "response.failed")]
Failed { response: OpenAIResponsesResponse },
/// Flat (`code`, `message`) from OpenAI, nested under `error` from some compatible servers.
#[serde(rename = "error")]
Error {
#[serde(default)]
code: Option<serde_json::Value>,
#[serde(default)]
message: Option<String>,
#[serde(default)]
error: Option<StreamErrorBody>,
},
/// Response created
#[serde(rename = "response.created")]
Created {},
@@ -890,6 +979,7 @@ pub struct OpenAIResponsesSSEParser {
pub usage: Option<OpenAIResponsesUsage>,
/// Reasoning summary parts seen so far, to separate them as paragraphs
reasoning_summary_parts: usize,
response_ended: bool,
}
impl OpenAIResponsesSSEParser {
@@ -906,11 +996,16 @@ impl OpenAIResponsesSSEParser {
used_websearch: false,
usage: None,
reasoning_summary_parts: 0,
response_ended: false,
}
}
}
impl SSEParser for OpenAIResponsesSSEParser {
fn is_complete(&self) -> bool {
self.response_ended
}
async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> {
let event: Option<OpenAIResponsesSSEEvent> = serde_json::from_str(data)
.inspect_err(|e| {
@@ -1010,11 +1105,22 @@ impl SSEParser for OpenAIResponsesSSEParser {
OpenAIResponsesSSEEvent::Completed { response }
| OpenAIResponsesSSEEvent::Incomplete { response } => {
self.response_ended = true;
if let Some(usage) = response.usage {
self.usage = Some(usage);
}
}
OpenAIResponsesSSEEvent::Failed { response } => {
return Err(response.error.unwrap_or_default().into_error("OpenAI"));
}
OpenAIResponsesSSEEvent::Error { code, message, error } => {
return Err(error
.unwrap_or(StreamErrorBody { code, message, ..Default::default() })
.into_error("OpenAI"));
}
OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => {
self.reasoning_summary_parts += 1;
if self.reasoning_summary_parts > 1 {
@@ -1037,9 +1143,10 @@ impl SSEParser for OpenAIResponsesSSEParser {
}
}
OpenAIResponsesSSEEvent::Done {} => self.response_ended = true,
// Ignore other event types
OpenAIResponsesSSEEvent::Done {}
| OpenAIResponsesSSEEvent::Created {}
OpenAIResponsesSSEEvent::Created {}
| OpenAIResponsesSSEEvent::InProgress {}
| OpenAIResponsesSSEEvent::OutputItemDone {}
| OpenAIResponsesSSEEvent::ContentPartAdded {}
@@ -1093,6 +1200,117 @@ mod tests {
}
}
fn sse_response(chunks: Vec<Result<&'static str, std::io::Error>>) -> Response {
let body = reqwest::Body::wrap_stream(futures::stream::iter(chunks));
Response::from(http::Response::new(body))
}
/// A stream that breaks off, or reports a failure inside its 200, must fail the call
/// rather than hand back what arrived as if it were the whole answer.
#[tokio::test]
async fn stream_failures_are_errors_not_partial_answers() {
use crate::retry::as_transient;
const STARTED: &str = concat!(
"data: {\"type\":\"message_start\"}\n\n",
"data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n",
"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hel\"}}\n\n",
);
let anthropic = |chunks| async move {
AnthropicSSEParser::new(Box::new(NoopSink))
.parse_events(sse_response(chunks))
.await
};
let overloaded = anthropic(vec![
Ok(STARTED),
Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n"),
])
.await
.unwrap_err();
assert!(as_transient(&overloaded).is_some(), "{overloaded}");
let refused = anthropic(vec![
Ok(STARTED),
Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"bad\"}}\n\n"),
])
.await
.unwrap_err();
assert!(as_transient(&refused).is_none(), "{refused}");
let truncated = anthropic(vec![Ok(STARTED)]).await.unwrap_err();
assert!(as_transient(&truncated).is_some(), "{truncated}");
let dropped = anthropic(vec![
Ok(STARTED),
Err(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"reset",
)),
])
.await
.unwrap_err();
assert!(as_transient(&dropped).is_some(), "{dropped}");
anthropic(vec![
Ok(STARTED),
Ok("data: {\"type\":\"message_stop\"}\n\n"),
])
.await
.unwrap();
anthropic(vec![
Ok(STARTED),
Ok("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n"),
])
.await
.unwrap();
let truncated = OpenAISSEParser::new(Box::new(NoopSink))
.parse_events(sse_response(vec![Ok(
"data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n",
)]))
.await
.unwrap_err();
assert!(as_transient(&truncated).is_some(), "{truncated}");
let quota = OpenAIResponsesSSEParser::new(Box::new(NoopSink))
.parse_events(sse_response(vec![Ok(
"data: {\"type\":\"error\",\"error\":{\"type\":\"insufficient_quota\",\"code\":\"insufficient_quota\",\"message\":\"m\"}}\n\n",
)]))
.await
.unwrap_err();
assert!(as_transient(&quota).is_none(), "{quota}");
GeminiSSEParser::new(Box::new(NoopSink))
.parse_events(sse_response(vec![Ok(
"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"name\":\"get_weather\",\"args\":{}}}]}}]}\n\n",
)]))
.await
.unwrap();
let gemini = |chunks| async move {
GeminiSSEParser::new(Box::new(NoopSink))
.parse_events(sse_response(chunks))
.await
};
// Thinking spent the whole `maxOutputTokens`: a clean close with no event.
gemini(vec![]).await.unwrap();
let dropped = gemini(vec![Err(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"reset",
))])
.await
.unwrap_err();
assert!(as_transient(&dropped).is_some(), "{dropped}");
let truncated = gemini(vec![Ok(
"data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Hi\"}]}}]}\n\n",
)])
.await
.unwrap_err();
assert!(as_transient(&truncated).is_some(), "{truncated}");
}
/// The prompt-side counts arrive only on `message_start` and the completion total
/// only on `message_delta`; a parser that keeps just the last one reports a request
/// with no input tokens at all.
backend/windmill-api-auth/src/scopes.rs
+47 -8
View File
@@ -170,10 +170,16 @@ const FLOW_JOBS: [&'static str; 6] = [
/// scoped by the agent's path like a deployed runnable's, as `jobs:run:agents:<path>`.
const AGENT_JOBS: [&'static str; 1] = ["jobs/run/agent"];
/// Lock resolution for scripts, apps and flows, scoped as `jobs:run:dependencies:<path>`.
/// Resolving a package can run its build or install scripts on the worker, so this is
/// still code execution: the scope only takes away the direct ways to run code.
const DEPENDENCY_JOBS: [&'static str; 2] = ["jobs/run/dependencies", "jobs/run/flow_dependencies"];
lazy_static::lazy_static! {
static ref RUN_PATH_ACTIONS: Vec<&'static str> = {
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component", "apps_u/upload_s3_file"];
let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "apps_u/execute_component", "apps_u/upload_s3_file"];
v.extend(DEPENDENCY_JOBS);
v.extend(SCRIPT_JOBS);
v.extend(FLOW_JOBS);
v.extend(AGENT_JOBS);
@@ -227,11 +233,13 @@ fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction {
}
}
/// Checks the route path to determine the runnable kind ("agents", "flows" or "scripts").
/// Checks the route path to determine the runnable kind ("dependencies", "agents", "flows"
/// or "scripts").
///
/// The order of checks is important:
/// - Flow-related paths are checked first to avoid false positives, as some flow paths
/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`).
/// - Dependency paths are checked before flow paths: `flow_dependencies` starts with `f`.
/// - Flow-related paths are checked before script paths to avoid false positives, as some
/// flow paths (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`).
///
/// Returns the kind based on the match, or `None` if no match is found.
fn determine_kind_from_route(route_path: &str) -> Option<String> {
@@ -247,6 +255,12 @@ fn determine_kind_from_route(route_path: &str) -> Option<String> {
{
return None;
}
if DEPENDENCY_JOBS
.iter()
.any(|path| route_path.starts_with(path))
{
return Some("dependencies".to_string());
}
if AGENT_JOBS.iter().any(|path| route_path.starts_with(path)) {
return Some("agents".to_string());
}
@@ -586,17 +600,23 @@ pub struct JobReadConfinement {
}
impl JobReadConfinement {
/// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows` or `agents`)
/// is inside the confinement. A read path admits every kind; a run scope only its own.
/// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows`, `agents` or
/// `dependencies`) is inside the confinement. A read path admits every kind; a run
/// scope only its own.
pub fn admits(&self, kind: &str, runnable_path: &str) -> bool {
if self.read_paths.allows(runnable_path) {
return true;
}
// Lock jobs and previews store the path their caller sent. The matcher reads `*`
// in it as a pattern, and `,`/`:` are scope syntax: such a path needs a grant
// covering every path, as enqueuing it did.
let resource = windmill_common::auth::is_scope_literal_path(runnable_path)
.then(|| vec![runnable_path.to_string()]);
let required = ScopeDefinition::new(
ScopeDomain::Jobs.as_str(),
ScopeAction::Run.as_str(),
Some(kind),
Some(vec![runnable_path.to_string()]),
resource,
);
self.run.iter().any(|scope| scope.includes(&required))
}
@@ -862,7 +882,8 @@ fn scope_grants_access(
/// from its own super-admin gate, these among them, so each is already readable by every
/// authenticated user; each is here because the CLI reads it from a job:
/// `automate_username_creation` before creating a user during a git-sync push, `uid` and
/// `hub_base_url` when `u/admin/hub_sync` pulls resource types from the Hub. The other
/// `hub_base_url` when a job runs `wmill hub pull`, as an edited copy of the retired
/// `u/admin/hub_sync` still does (servers sync resource types themselves). The other
/// ungated keys have no such caller, so they stay confined — being ungated earns a key
/// nothing on its own. Listing a gated key earns it nothing either: `require_super_admin`
/// refuses every job token, so `hub_api_secret`, which that pull reads for a private Hub,
@@ -1714,6 +1735,24 @@ mod tests {
scope_for_route("POST", "/api/w/ws/jobs/run/agent/u/x/y").as_deref(),
Some("jobs:run:agents")
);
for path in [
"/api/w/ws/jobs/run/dependencies",
"/api/w/ws/jobs/run/dependencies_async",
"/api/w/ws/jobs/run/flow_dependencies",
"/api/w/ws/jobs/run/flow_dependencies_async",
] {
assert_eq!(
scope_for_route("POST", path).as_deref(),
Some("jobs:run:dependencies"),
"{path}"
);
assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_ok());
assert!(check_route_access(&["jobs:run:flows".into()], path, "POST").is_err());
}
// A dependencies grant reaches no other run route.
for path in ["/api/w/ws/jobs/run/preview", "/api/w/ws/jobs/run/p/u/x/y"] {
assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_err());
}
// Preview/bundle runs have no deployed path and their handlers require the
// broad `jobs:run` scope, so the derived scope must not carry a kind.
backend/windmill-api-flows/src/flows.rs
+35 -6
View File
@@ -20,7 +20,8 @@ use windmill_api_auth::{
require_owner_of_path, ApiAuthed,
};
use windmill_common::workspaces::{
check_deploy_rules, check_operator_can_build_flows, operator_can_build_flows, RuleCheckResult,
check_deploy_rules, check_operator_can_build, operator_builder_rights, BuilderKind,
RuleCheckResult,
};
use windmill_common::{
user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay},
@@ -243,7 +244,7 @@ async fn list_flows(
// Append the authed user's drafts at paths with no deployed flow; see scripts.rs.
if lq.include_draft_only.unwrap_or(false)
&& (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?)
&& (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.flows)
&& offset == 0
&& lq.path_start.is_none()
&& lq.path_exact.is_none()
@@ -731,7 +732,14 @@ async fn create_flow(
Path(w_id): Path<String>,
Json(mut nf): Json<NewFlow>,
) -> Result<(StatusCode, String)> {
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "create flows").await?;
check_operator_can_build(
&db,
&w_id,
authed.is_operator,
BuilderKind::Flows,
"create flows",
)
.await?;
check_scopes(&authed, || format!("flows:write:{}", nf.path))?;
// A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly.
@@ -1344,7 +1352,14 @@ async fn update_flow(
Path((w_id, flow_path)): Path<(String, StripPath)>,
Json(ef): Json<EditFlow>,
) -> Result<String> {
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "update flows").await?;
check_operator_can_build(
&db,
&w_id,
authed.is_operator,
BuilderKind::Flows,
"update flows",
)
.await?;
let flow_path = flow_path.to_path();
// The URL identifies the flow being updated; the body path is only needed to rename.
let mut nf = ef.into_new_flow(flow_path);
@@ -2038,7 +2053,14 @@ async fn archive_flow_by_path(
Path((w_id, path)): Path<(String, StripPath)>,
Json(archived): Json<Archived>,
) -> Result<String> {
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "archive flows").await?;
check_operator_can_build(
&db,
&w_id,
authed.is_operator,
BuilderKind::Flows,
"archive flows",
)
.await?;
let path = path.to_path();
check_scopes(&authed, || format!("flows:write:{}", path))?;
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
@@ -2179,7 +2201,14 @@ async fn delete_flow_by_path(
Path((w_id, path)): Path<(String, StripPath)>,
Query(query): Query<DeleteFlowQuery>,
) -> Result<String> {
check_operator_can_build_flows(&db, &w_id, authed.is_operator, "delete flows").await?;
check_operator_can_build(
&db,
&w_id,
authed.is_operator,
BuilderKind::Flows,
"delete flows",
)
.await?;
let path = path.to_path();
check_scopes(&authed, || format!("flows:write:{}", path))?;
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
backend/windmill-api-integration-tests/tests/operator_builder_apps.rs
+570
View File
@@ -0,0 +1,570 @@
use serde_json::json;
use serial_test::serial;
use sqlx::{Pool, Postgres};
use windmill_common::workspaces::invalidate_operator_rights_cache;
use windmill_test_utils::*;
/// Every test here must be `#[serial]`: the rights cache is process-global and keyed by workspace id
/// alone, so parallel tests under this same id read each other's rights.
const WS: &str = "test-workspace";
fn client(token: &str) -> reqwest::Client {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
reqwest::header::AUTHORIZATION,
reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")).unwrap(),
);
reqwest::ClientBuilder::new()
.default_headers(headers)
.build()
.unwrap()
}
async fn set_builder(db: &Pool<Postgres>, flows: bool, apps: bool) -> anyhow::Result<()> {
sqlx::query(
"UPDATE workspace_settings SET operator_settings = $1::text::jsonb WHERE workspace_id = $2",
)
.bind(format!(
r#"{{"builder_flows": {flows}, "builder_apps": {apps}}}"#
))
.bind(WS)
.execute(db)
.await?;
// The rights are read through a process-global 60s cache keyed by workspace id.
invalidate_operator_rights_cache(WS);
Ok(())
}
async fn add_script(db: &Pool<Postgres>, hash: i64, path: &str, owner: &str) -> anyhow::Result<()> {
sqlx::query(
"INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema,
summary, description, lock, extra_perms)
VALUES ($1, $2, $3, 'x', 'bun', 'script', $4, '{}', '', '', '', '{}')",
)
.bind(WS)
.bind(hash)
.bind(path)
.bind(owner)
.execute(db)
.await?;
Ok(())
}
/// A full-code app is deployed multipart, the way the editor and the CLI use it.
fn raw_app_form(app: serde_json::Value) -> reqwest::multipart::Form {
reqwest::multipart::Form::new()
.part(
"app",
reqwest::multipart::Part::text(app.to_string())
.mime_str("application/json")
.unwrap(),
)
.part(
"js",
reqwest::multipart::Part::text("console.log(1)").file_name("app.js"),
)
}
fn raw_app(path: &str, mode: &str, runnable_path: &str) -> reqwest::multipart::Form {
raw_app_form(json!({
"path": path,
"summary": "",
"value": {"files": {}, "runnables": {"r": {
"name": "r", "type": "runnableByPath", "runType": "script",
"path": runnable_path
}}},
"policy": {"execution_mode": mode, "triggerables_v2": {}}
}))
}
fn composition_flow(path: &str) -> serde_json::Value {
json!({
"path": path, "summary": "", "description": "", "schema": {},
"value": {"modules": [{
"id": "a",
"value": {"type": "script", "path": "u/operator/some_script", "input_transforms": {}}
}]}
})
}
/// The app half of the builder boundary, plus the proof that the two rights are independent: a
/// gate reading "either" would authorize the kind its workspace never granted, which is the whole
/// point of splitting them.
#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))]
#[serial]
async fn test_operator_builder_apps_boundary(db: Pool<Postgres>) -> anyhow::Result<()> {
initialize_tracing().await;
let server = ApiServer::start(db.clone()).await?;
let port = server.addr.port();
let api = format!("http://localhost:{port}/api/w/{WS}");
let c = client("OPERATOR_TOKEN_1");
add_script(&db, 4241, "u/operator/some_script", "operator").await?;
// `permissions_test` gives the operator fixture no rights on `u/alice/**`.
add_script(&db, 4243, "u/alice/private", "alice").await?;
// A version shared with the operator, then archived, and a newer one at the same path that is
// not: the newer one is what runs.
add_script(&db, 4244, "u/alice/reused", "alice").await?;
sqlx::query(
"UPDATE script SET archived = true, extra_perms = '{\"u/operator\": false}',
created_at = now() - interval '1 day' WHERE hash = 4244",
)
.execute(&db)
.await?;
add_script(&db, 4245, "u/alice/reused", "alice").await?;
set_builder(&db, false, false).await?;
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app(
"u/operator/a0",
"publisher",
"u/operator/some_script",
))
.send()
.await?;
assert!(
!resp.status().is_success(),
"an operator without the builder right must not create a full-code app"
);
set_builder(&db, true, true).await?;
// `*_raw_source` compiles caller-supplied sources with a bundler job on a worker. It sits
// beside `create_app_raw`, which builders MAY use, so it is the gate most likely to be opened
// by mistake later.
let resp = c
.post(format!("{api}/apps/create_raw_source"))
.json(&json!({
"path": "u/operator/a1", "summary": "",
"value": {"files": {"index.ts": "console.log(1)"}, "runnables": {}},
"policy": {"execution_mode": "publisher"}
}))
.send()
.await?;
assert!(
!resp.status().is_success(),
"a builder must not compile app sources on a worker"
);
// Low-code apps carry inline scripts, so they stay shut too.
let resp = c
.post(format!("{api}/apps/create"))
.json(&json!({
"path": "u/operator/a2", "summary": "",
"value": {"grid": []}, "policy": {"execution_mode": "publisher"}
}))
.send()
.await?;
assert!(
!resp.status().is_success(),
"a builder must not create a low-code app"
);
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app("u/operator/a3", "publisher", "u/alice/private"))
.send()
.await?;
assert!(
!resp.status().is_success(),
"a builder must not deploy an app referencing a runnable it cannot read"
);
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app("u/operator/a4", "viewer", "u/operator/some_script"))
.send()
.await?;
assert!(
!resp.status().is_success(),
"a builder must not deploy a viewer-mode app: the policy stops bounding what it can invoke"
);
// Storage strips a NUL, turning this key into `inlineScript`.
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app_form(json!({
"path": "u/operator/a_nul", "summary": "",
"value": {"files": {}, "runnables": {"r": {
"type": "inline",
"inline\u{0}Script": {"content": "x", "language": "bun"}
}}},
"policy": {"execution_mode": "publisher", "triggerables_v2": {}}
})))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"a NUL in a key must not hide an inline script: {}",
resp.text().await?
);
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app(
"u/operator/a5",
"publisher",
"u/operator/some_script",
))
.send()
.await?;
assert!(
resp.status().is_success(),
"a builder must be able to deploy a full-code app over readable runnables: {}",
resp.text().await?
);
let sandbox: Option<bool> = sqlx::query_scalar(
"SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2",
)
.bind(WS)
.bind("u/operator/a5")
.fetch_one(&db)
.await?;
assert_eq!(
sandbox,
Some(true),
"a builder-authored app must be stored sandboxed"
);
// `execute_component` looks up `<component>:<path>` with an unrestricted component string, so
// a key with a second colon still resolves at run time and must not slip past validation.
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app_form(json!({
"path": "u/operator/a6", "summary": "",
"value": {"files": {}, "runnables": {}},
"policy": {"execution_mode": "publisher", "triggerables_v2": {
"x:y:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}}
}}
})))
.send()
.await?;
assert!(
!resp.status().is_success(),
"a multi-colon triggerable key must not hide an unreadable runnable from validation"
);
// The two rights are independent.
set_builder(&db, true, false).await?;
let resp = c
.post(format!("{api}/flows/create"))
.json(&composition_flow("u/operator/flows_only"))
.send()
.await?;
assert!(
resp.status().is_success(),
"flows-only must still create a flow: {}",
resp.text().await?
);
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app(
"u/operator/flows_only_app",
"publisher",
"u/operator/some_script",
))
.send()
.await?;
assert!(
!resp.status().is_success(),
"flows-only must not deploy a full-code app"
);
set_builder(&db, false, true).await?;
let resp = c
.post(format!("{api}/flows/create"))
.json(&composition_flow("u/operator/apps_only"))
.send()
.await?;
assert!(
!resp.status().is_success(),
"apps-only must not create a flow"
);
let resp = c
.post(format!("{api}/apps/create_raw"))
.multipart(raw_app(
"u/operator/apps_only_app",
"publisher",
"u/operator/some_script",
))
.send()
.await?;
assert!(
resp.status().is_success(),
"apps-only must still deploy a full-code app: {}",
resp.text().await?
);
// The deploy panel saves access and frontend scopes through the settings-only endpoint.
for (body, expected) in [
(
json!({"policy": {"execution_mode": "anonymous", "triggerables_v2": {}}}),
200,
),
(
json!({"policy": {"execution_mode": "publisher", "sandbox": false, "triggerables_v2": {}}}),
403,
),
(
json!({"value": {"files": {}, "runnables": {}}, "policy": {"execution_mode": "publisher"}}),
403,
),
] {
let resp = c
.post(format!("{api}/apps/update/u/operator/apps_only_app"))
.json(&body)
.send()
.await?;
let status = resp.status();
assert_eq!(
status,
expected,
"settings update {body}: {}",
resp.text().await?
);
}
let sandbox: Option<bool> = sqlx::query_scalar(
"SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2",
)
.bind(WS)
.bind("u/operator/apps_only_app")
.fetch_one(&db)
.await?;
assert_eq!(
sandbox,
Some(true),
"a settings update must not turn a builder app's sandbox off"
);
// The editor runs the runnables a draft names as whoever opens it, and its frontend code
// with their session unless the preview is sandboxed.
for (runnables, policy, expected) in [
(json!({}), json!({"sandbox": true}), 200),
(
json!({"a": {"type": "inline", "inlineScript": {"content": "x", "language": "bun"}}}),
json!({"sandbox": true}),
403,
),
(json!({}), json!({"sandbox": false}), 403),
(
json!({}),
json!({"sandbox": true, "execution_mode": "viewer"}),
403,
),
(json!({}), json!(null), 403),
(
json!({}),
json!({"sandbox": true, "on_behalf_of": "u/alice", "on_behalf_of_email": "alice@windmill.dev"}),
403,
),
(
json!({}),
json!({"sandbox": true, "on_behalf_of": "u/operator"}),
200,
),
(
json!({}),
json!({"sandbox": true, "triggerables": {
format!("{}script/u/operator/some_script", "a:".repeat(300)): {}
}}),
400,
),
(
json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/operator/some_script"}}),
json!({"sandbox": true}),
200,
),
(
json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/private"}}),
json!({"sandbox": true}),
403,
),
(
json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/reused"}}),
json!({"sandbox": true}),
403,
),
(
json!({"r": {"type": "path", "runType": "hubscript", "path": "hub/1/x/y"}}),
json!({"sandbox": true}),
403,
),
(
json!({"r": {"type": "path", "runType": "script/u/alice", "path": "u/operator/some_script"}}),
json!({"sandbox": true}),
400,
),
(
json!({"r": {"type": "path", "runType": "script", "path": ["u/alice/private"]}}),
json!({"sandbox": true}),
400,
),
(
json!({"r": {"type": ["path"], "runType": "script", "path": "u/alice/private"}}),
json!({"sandbox": true}),
400,
),
(
json!([{"type": "path", "runType": "script", "path": "u/alice/private"}]),
json!({"sandbox": true}),
400,
),
(
json!({"r": {"type": "path", "runType": "script", "path": "u/operator/some_script",
"fields": {"x": {"type": "static", "value": {"type": "path", "path": "/srv/out"}}}}}),
json!({"sandbox": true}),
200,
),
(
json!({}),
json!({"sandbox": true, "triggerables": {"x:rawscript/abc": {}}}),
403,
),
(
json!({}),
json!({"sandbox": true, "triggerables": {"x:script/u/alice/private": {}}}),
403,
),
(
json!({}),
json!({"sandbox": true, "triggerables_v2": {
"x:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}},
"y:rawscript/abc": {"static_inputs": {}, "one_of_inputs": {}}
}}),
200,
),
] {
let resp = c
.post(format!("{api}/drafts/update/raw_app/u/operator/d1"))
.json(&json!({
"value": {"files": {}, "runnables": runnables, "summary": "", "policy": policy},
"force": true
}))
.send()
.await?;
let status = resp.status();
assert_eq!(
status,
expected,
"raw app draft {runnables} {policy}: {}",
resp.text().await?
);
}
// The component preview runs a deployed runnable for a builder; inline code and hub
// scripts are unreviewed code.
for (preview, expected) in [
(json!({"path": "script/u/operator/some_script"}), 200),
(json!({"path": "script/hub/1/x/y"}), 403),
(
json!({"raw_code": {"content": "x", "language": "bun"}}),
403,
),
] {
let mut body = json!({"component": "c", "args": {}, "force_viewer_static_fields": {}});
body.as_object_mut()
.unwrap()
.extend(preview.as_object().unwrap().clone());
let resp = c
.post(format!(
"{api}/apps_u/execute_component/u/operator/apps_only_app"
))
.json(&body)
.send()
.await?;
let status = resp.status();
assert_eq!(
status,
expected,
"component preview {preview}: {}",
resp.text().await?
);
}
// Delete and redeploy each serve both kinds; builder rights cover only full-code apps.
let resp = client("SECRET_TOKEN")
.post(format!("{api}/apps/create"))
.json(&json!({
"path": "u/operator/low_code", "summary": "",
"value": {"grid": []}, "policy": {"execution_mode": "publisher"}
}))
.send()
.await?;
assert!(resp.status().is_success(), "{}", resp.text().await?);
let resp = c
.post(format!("{api}/apps/update_raw/u/operator/low_code"))
.multipart(raw_app_form(json!({
"policy": {"execution_mode": "publisher", "triggerables_v2": {}}
})))
.send()
.await?;
assert_eq!(
resp.status(),
403,
"policy-only update of a low-code app: {}",
resp.text().await?
);
// Deployment history restores a full-code version with allow_kind_change set.
let resp = c
.post(format!("{api}/apps/update_raw/u/operator/apps_only_app"))
.multipart(raw_app_form(json!({
"value": {"files": {}, "runnables": {}},
"policy": {"execution_mode": "publisher", "triggerables_v2": {}},
"allow_kind_change": true
})))
.send()
.await?;
assert_eq!(
resp.status(),
200,
"restore of a full-code version: {}",
resp.text().await?
);
for (path, expected) in [("low_code", 403), ("apps_only_app", 200)] {
let resp = c
.delete(format!("{api}/apps/delete/u/operator/{path}"))
.send()
.await?;
let status = resp.status();
assert_eq!(status, expected, "delete {path}: {}", resp.text().await?);
}
Ok(())
}
/// The readability check must not hold its RLS transaction while it takes a second connection,
/// or a one-connection pool (`DATABASE_CONNECTIONS=1`) stalls every builder draft naming a script.
#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))]
#[serial]
async fn test_operator_builder_app_check_fits_one_connection(
pool_opts: sqlx::postgres::PgPoolOptions,
connect_opts: sqlx::postgres::PgConnectOptions,
) -> anyhow::Result<()> {
initialize_tracing().await;
let db = pool_opts
.max_connections(1)
.acquire_timeout(std::time::Duration::from_secs(5))
.connect_with(connect_opts)
.await?;
let server = ApiServer::start(db.clone()).await?;
let api = format!("http://localhost:{}/api/w/{WS}", server.addr.port());
add_script(&db, 4241, "u/operator/some_script", "operator").await?;
set_builder(&db, false, true).await?;
let resp = client("OPERATOR_TOKEN_1")
.post(format!("{api}/drafts/update/raw_app/u/operator/d1"))
.json(&json!({
"value": {"files": {}, "summary": "", "runnables": {"r": {
"name": "r", "type": "runnableByPath", "runType": "script",
"path": "u/operator/some_script"
}}, "policy": {"sandbox": true, "triggerables_v2": {}}},
"force": true
}))
.send()
.await?;
let status = resp.status();
assert_eq!(status, 200, "{}", resp.text().await?);
Ok(())
}
backend/windmill-api-jobs/src/execution.rs
+25 -2
View File
@@ -245,6 +245,27 @@ pub async fn unclaimable_run_lineage(
Ok(referenced)
}
/// `WM_END_USER_EMAIL` of a run started through a `/jobs/run*` route. `WM_EMAIL` names the
/// runnable's `on_behalf_of` identity when it has one, so this is the only place the caller's
/// address reaches the job. A job's own `WM_TOKEN` passes on the end user of that job, and a run
/// a trigger fired has no end user.
pub async fn run_end_user_email(
db: &DB,
w_id: &str,
authed: &ApiAuthed,
trigger: Option<&TriggerMetadata>,
) -> error::Result<Option<String>> {
if trigger.is_some() {
return Ok(None);
}
match authed.job_id {
Some(job_id) => Ok(windmill_common::auth::get_job_perms(db, &job_id, w_id)
.await?
.and_then(|p| p.end_user_email)),
None => Ok(Some(authed.email.clone())),
}
}
#[cfg(feature = "enterprise")]
pub async fn check_license_key_valid() -> error::Result<()> {
use windmill_common::ee_oss::LICENSE_KEY_VALID;
@@ -1013,6 +1034,7 @@ pub async fn run_flow<'c>(
)
};
let end_user_email = run_end_user_email(db, w_id, authed, trigger.as_ref()).await?;
let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?;
let (uuid, mut tx) = push(
&db,
@@ -1048,7 +1070,7 @@ pub async fn run_flow<'c>(
None,
push_authed.as_ref(),
false,
None,
end_user_email,
authed.trigger_or_fallback(trigger),
run_query.suspended_mode,
scope_ceiling.as_deref(),
@@ -1244,6 +1266,7 @@ pub async fn push_script_job_by_path_into_queue<'c>(
)
};
let end_user_email = run_end_user_email(&db, &w_id, &authed, trigger.as_ref()).await?;
let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?;
let (uuid, tx) = push(
&db,
@@ -1277,7 +1300,7 @@ pub async fn push_script_job_by_path_into_queue<'c>(
},
push_authed.as_ref(),
false,
None,
end_user_email,
authed.trigger_or_fallback(trigger),
run_query.suspended_mode,
scope_ceiling.as_deref(),
backend/windmill-api-scripts/src/scripts.rs
-12
View File
@@ -4266,12 +4266,6 @@ async fn delete_script_by_path(
check_scopes(&authed, || format!("scripts:write:{}", path))?;
if path == "u/admin/hub_sync" && w_id == "admins" {
return Err(Error::BadRequest(
"Cannot delete the global setup app".to_string(),
));
}
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
&w_id,
AuditAuthorable::username(&authed),
@@ -4445,12 +4439,6 @@ async fn delete_scripts_bulk(
require_admin(authed.is_admin, &authed.username)?;
if request.paths.contains(&"u/admin/hub_sync".to_string()) && w_id == "admins" {
return Err(Error::BadRequest(
"Cannot delete the global setup app".to_string(),
));
}
if let RuleCheckResult::Blocked(msg) = check_deploy_rules(
&w_id,
AuditAuthorable::username(&authed),
backend/windmill-api-settings/src/lib.rs
+41 -202
View File
@@ -30,6 +30,7 @@ mod storage_usage;
use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed};
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::hub_resource_types::{self, SyncSource};
use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT;
use windmill_common::DB;
@@ -58,10 +59,10 @@ use windmill_common::{
error::{self, pg_error_message, JsonResult, Result},
get_database_url,
global_settings::{
ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING,
AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS,
EXTERNAL_INSTANCE_PG_SETTING,
ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING,
APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING,
CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING,
DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING,
GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING,
HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING,
INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES,
@@ -1238,26 +1239,22 @@ async fn run_setting_pre_write_hook(
}
}
}
GITHUB_APP_WEBHOOK_BASE_URL_SETTING => {
// A bad value here yields a webhook GitHub can never deliver to, and the
GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => {
// A bad webhook base yields a webhook GitHub can never deliver to, and the
// failure only shows up much later as "falling back to polling" on a
// repository — so reject it at the boundary instead.
// repository; a bad API base puts an unreachable URL in front of every user.
// So reject both at the boundary instead.
match value {
// Clearing (delete row) is handled by the caller; allow it through.
serde_json::Value::Null => {}
serde_json::Value::String(s) if s.trim().is_empty() => {}
serde_json::Value::String(s) => {
windmill_common::global_settings::validate_webhook_base_url(s).map_err(
|e| {
error::Error::BadRequest(format!(
"{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}"
))
},
)?;
windmill_common::global_settings::validate_webhook_base_url(s)
.map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?;
}
_ => {
return Err(error::Error::BadRequest(format!(
"{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string"
"{key} must be a URL string"
)));
}
}
@@ -1394,8 +1391,12 @@ async fn set_instance_config(
// Applied and audited one setting at a time: the writes are not transactional, so a
// batch that fails midway must not leave the settings it did write unrecorded.
let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } =
settings_diff;
let instance_config::SettingsDiff {
upserts,
deletes,
mut previous_values,
unchanged_count,
} = settings_diff;
let steps = upserts
.into_iter()
.map(|(k, v)| (k, Some(v)))
@@ -2576,95 +2577,6 @@ pub async fn get_jwks() -> JsonResult<JwksResponse> {
Ok(Json(JwksResponse { keys: vec![] }))
}
#[derive(serde::Deserialize, serde::Serialize)]
struct CachedResourceType {
#[allow(dead_code)]
id: i64,
name: String,
schema: Option<serde_json::Value>,
#[allow(dead_code)]
app: String,
description: Option<String>,
/// Doubly optional, and read through a wrapping deserializer: this struct also
/// decodes the on-disk cache, where an absent key means "written before the
/// column, leave the stored extension alone" and an explicit null means the hub
/// dropped it. Plain serde folds both into `None`.
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option"
)]
format_extension: Option<Option<String>>,
/// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit
/// null (the hub naming nothing) clears it.
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option"
)]
display_name: Option<Option<String>>,
}
#[derive(serde::Deserialize)]
struct HubResourceTypeRaw {
id: i64,
name: String,
schema: Option<String>,
app: String,
description: Option<String>,
#[serde(default)]
format_extension: Option<String>,
#[serde(
default,
deserialize_with = "windmill_common::more_serde::double_option"
)]
display_name: Option<Option<String>>,
}
async fn fetch_resource_types_from_hub() -> error::Result<Vec<CachedResourceType>> {
let response = HTTP_CLIENT
.get(format!(
"{}/resource_types/list",
windmill_common::DEFAULT_HUB_BASE_URL
))
.header("Accept", "application/json")
.send()
.await
.map_err(|e| error::Error::InternalErr(format!("Failed to fetch from hub: {}", e)))?;
if !response.status().is_success() {
return Err(error::Error::InternalErr(format!(
"Hub returned status {}",
response.status()
)));
}
let raw_types: Vec<HubResourceTypeRaw> = response
.json()
.await
.map_err(|e| error::Error::InternalErr(format!("Failed to parse hub response: {}", e)))?;
Ok(raw_types
.into_iter()
.filter_map(|rt| {
let schema = match rt.schema {
Some(s) => match serde_json::from_str(&s) {
Ok(v) => Some(v),
Err(_) => return None,
},
None => None,
};
Some(CachedResourceType {
id: rt.id,
name: rt.name,
schema,
app: rt.app,
description: rt.description,
format_extension: Some(rt.format_extension),
display_name: rt.display_name,
})
})
.collect())
}
#[derive(serde::Deserialize)]
struct SyncResourceTypesQuery {
name: Option<String>,
@@ -2677,118 +2589,45 @@ async fn sync_cached_resource_types(
) -> error::Result<String> {
require_super_admin(&db, &authed).await?;
use windmill_common::worker::HUB_RT_CACHE_DIR;
let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR);
let outcome = hub_resource_types::sync(&db).await?;
// Manual sync is hub-first so it lands newly-published hub types on demand. The
// on-disk cache is only a fallback for when the hub is unreachable (airgapped
// installs / network error); refreshing it is left to the daily cache-rt cron and
// the startup sync in main.rs, which own the offline path.
let (resource_types, from_hub) = match fetch_resource_types_from_hub().await {
Ok(types) => {
tracing::info!("Fetched {} resource types live from the hub", types.len());
(types, true)
}
Err(hub_err) => {
tracing::warn!(
"Live hub fetch failed ({hub_err}), falling back to on-disk cache at {cache_path}"
);
match tokio::fs::read_to_string(&cache_path).await {
Ok(content) => {
let parsed = serde_json::from_str::<Vec<CachedResourceType>>(&content)
.map_err(|e| {
error::Error::InternalErr(format!(
"Failed to parse cached resource types: {}",
e
))
})?;
(parsed, false)
}
Err(_) => return Err(hub_err),
}
}
};
let mut synced_count = 0;
for rt in &resource_types {
// A name too long for the column counts as absent, leaving the stored one alone: one bad
// entry must not fail the upsert and end the rest of the sync.
let display_name = match &rt.display_name {
Some(Some(name)) if name.chars().count() > 100 => None,
other => other.clone(),
if !outcome.changed.is_empty() {
let source = match outcome.source {
SyncSource::Hub => "hub",
SyncSource::ImageCache => "image_cache",
};
let exists: Option<bool> = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))",
&rt.name,
rt.schema.as_ref(),
rt.description.as_deref(),
rt.format_extension.clone().flatten(),
rt.format_extension.is_some(),
display_name.clone().flatten(),
display_name.is_some(),
let changed = outcome.changed.join(", ");
audit_log(
&db,
&authed,
"resource_types.sync",
ActionKind::Update,
"admins",
None,
Some([("source", source), ("changed", changed.as_str())].into()),
)
.fetch_one(&db)
.await?;
if exists.unwrap_or(false) {
continue;
}
sqlx::query!(
// Whether the payload carried the key at all is what decides: present
// (even as null) is authoritative and may clear, absent means a cache
// written before the column and must leave the stored value alone.
"INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)
VALUES ('admins', $1, $2, $3, $4, $6, now())
ON CONFLICT (workspace_id, name) DO UPDATE
SET schema = EXCLUDED.schema, description = EXCLUDED.description,
-- A fileset is a set of files, so it cannot also be one file.
-- Create and update reject the pair; this writer bypasses both, so
-- it declines the extension rather than persisting the forbidden
-- combination onto a same-named local fileset.
format_extension = CASE
WHEN resource_type.is_fileset THEN NULL
WHEN $5 THEN EXCLUDED.format_extension
ELSE resource_type.format_extension END,
display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,
edited_at = now()",
&rt.name,
rt.schema.as_ref(),
rt.description.as_deref(),
rt.format_extension.clone().flatten(),
rt.format_extension.is_some(),
display_name.clone().flatten(),
display_name.is_some(),
)
.execute(&db)
.await?;
synced_count += 1;
}
// If a specific type was requested and is still absent after syncing, surface an
// explicit not-found instead of a silent "Synced 0". Word it by source so the
// cache-fallback path does not claim it checked the hub.
if let Some(name) = name.as_deref() {
if !resource_types.iter().any(|rt| rt.name == name) {
let source = if from_hub {
"on the hub"
} else {
"in the cached resource types (hub unreachable)"
if !outcome.listed.iter().any(|listed| listed == name) {
let source = match outcome.source {
SyncSource::Hub => "on the hub",
SyncSource::ImageCache => "in the cached resource types (hub unavailable)",
};
return Err(error::Error::NotFound(format!(
"resource type '{}' not found {}",
name, source
"resource type '{name}' not found {source}"
)));
}
}
Ok(format!(
"Synced {} resource types ({} unchanged)",
synced_count,
resource_types.len() - synced_count
))
Ok(match outcome.source {
SyncSource::Hub => outcome.summary(),
SyncSource::ImageCache => format!("{}, the hub being unavailable", outcome.summary()),
})
}
#[cfg(test)]
backend/windmill-api-users/src/users.rs
+1 -1
View File
@@ -490,7 +490,7 @@ async fn list_user_usage(
"
SELECT usr.email, usage.executions
FROM usr, LATERAL (
SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions
SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions
FROM v2_job_completed c JOIN v2_job j USING (id)
WHERE j.workspace_id = $1
AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')

Some files were not shown because too many files have changed in this diff Show More