diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 87a65214ab..2ce47ea714 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv ENV TZ=Etc/UTC diff --git a/.github/workflows/ai-evals-scheduled.yml b/.github/workflows/ai-evals-scheduled.yml index 4780eb02eb..580a757c21 100644 --- a/.github/workflows/ai-evals-scheduled.yml +++ b/.github/workflows/ai-evals-scheduled.yml @@ -16,7 +16,7 @@ on: default: "global cli" models: description: "Space-separated model aliases (bun run cli -- models); cli skips those it has no agent for" - default: "sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro" + default: "sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro" runs: description: "Runs per case" default: "3" @@ -28,7 +28,7 @@ concurrency: group: ai-evals-scheduled-${{ github.ref }} env: - MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6-sol gpt-6-astra deepseek-v4-pro' }} + MODELS: ${{ inputs.models || 'sonnet-5.5 opus-5.5 gpt-6.1-sol gpt-6-astra deepseek-v4-pro' }} RUNS: ${{ inputs.runs || '3' }} REASONING: ${{ inputs.reasoning }} # Routes on the cloud instance are prefixed with their workspace. diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index 54a62ec495..3468449753 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -63,7 +63,7 @@ jobs: - uses: denoland/setup-deno@v2 with: - # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI + # Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 @@ -81,7 +81,7 @@ jobs: - uses: astral-sh/setup-uv@v7 with: - version: "0.11.24" + version: "0.13.0" - uses: shivammathur/setup-php@v2 with: diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index a2087025b7..73f955131e 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -50,7 +50,7 @@ jobs: dotnet-version: "9.0.x" - uses: denoland/setup-deno@v2 with: - # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI + # Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - uses: actions/setup-go@v7 @@ -64,7 +64,7 @@ jobs: node-version: "24" - uses: astral-sh/setup-uv@v7 with: - version: "0.11.24" + version: "0.13.0" - uses: shivammathur/setup-php@v2 with: php-version: "8.3" @@ -295,4 +295,4 @@ jobs: # never reaches it. Pin the target dir (matching the cache step above) so # its own tests run off this compile rather than a second bundled build. (cd windmill-duckdb-ffi-internal && export CARGO_TARGET_DIR="$PWD/target" && ./build_dev.sh && cargo test --release -p windmill_duckdb_ffi_internal) - DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger --all -- --nocapture --test-threads=10 + DENO_PATH=$(which deno) BUN_PATH=$(which bun) NODE_BIN_PATH=$(which node) GO_PATH=$(which go) UV_PATH=$(which uv) PHP_PATH=$(which php) COMPOSER_PATH=$(which composer) RUBY_PATH=$(which ruby) RUBY_BUNDLE_PATH=$(which bundle) RUBY_GEM_PATH=$(which gem) POWERSHELL_PATH=$(which pwsh) DOTNET_PATH=$(which dotnet) cargo test --features enterprise,deno_core,duckdb,license,python,rust,scoped_cache,parquet,private,private_registry_test,csharp,php,ruby,mysql,quickjs,mcp,run_inline,smtp,http_trigger,openidconnect --all -- --nocapture --test-threads=10 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d420ff1f00..475d7860d8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,7 +27,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -111,7 +111,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -125,7 +125,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -136,7 +136,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -147,7 +147,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -188,7 +188,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -202,7 +202,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -213,7 +213,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -224,7 +224,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -235,7 +235,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -246,7 +246,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -257,7 +257,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -268,7 +268,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always steps: @@ -309,7 +309,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl diff --git a/.github/workflows/npm_on_release.yml b/.github/workflows/npm_on_release.yml index 631ab53153..a2bd974daa 100644 --- a/.github/workflows/npm_on_release.yml +++ b/.github/workflows/npm_on_release.yml @@ -18,16 +18,19 @@ jobs: env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_chat_sdk: - runs-on: ubicloud-standard-8 + # Published by npm trusted publishing (OIDC), which npm accepts only from + # GitHub-hosted runners and npm >= 11.5.1: keep both, there is no token to fall back on. + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v7 with: - node-version: "20.x" + node-version: "24.x" registry-url: "https://registry.npmjs.org" - run: cd chat-sdk && npm ci && npm run build && npm publish --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} publish_cli: runs-on: ubicloud-standard-8 steps: diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 3f035f45c0..fdce702331 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.826.0" + ".": "1.830.0" } diff --git a/AGENTS.md b/AGENTS.md index e4915cdcb6..a57215cde5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,12 +33,16 @@ Open-source platform for internal tools, workflows, API integrations, background administers behind `external_instance` data tables and Ducklake catalogs: its invariants (one lifecycle lock, managed-object markers, the setup gate, per-cluster roles, fork copy ownership) and how to run one locally +- **Data table viewer roles**: `docs/datatable-viewer-roles.md`, on what a connection naming no + role holds: the default role as a floor, where the viewer comes from, why an admin's roles are + not the viewer's, the per-role-set logins and the order their rows and logins are written in - **Enterprise**: `docs/enterprise.md` — EE file conventions and PR workflow - **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option` fields and a jsonb merge rather than a serde default -- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace setting that lets - operators compose flows, what the composition check must cover, and why it costs a full seat +- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace settings that let + operators compose flows and full-code apps, what the composition check must cover, which + raw-app endpoint stays closed on purpose, and why either right costs a full seat - **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ce5ffcde3..80939b1b47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,98 @@ # Changelog +## [1.830.0](https://github.com/windmill-labs/windmill/compare/v1.829.0...v1.830.0) (2026-10-09) + + +### Features + +* add a jobs:run:dependencies token scope for lock generation ([#11643](https://github.com/windmill-labs/windmill/issues/11643)) ([1dc4ed5](https://github.com/windmill-labs/windmill/commit/1dc4ed5e7fb86c9658f1f877e7b92cd9dab5e283)) +* let operators compose full-code apps when the workspace grants it ([#11229](https://github.com/windmill-labs/windmill/issues/11229)) ([02e4697](https://github.com/windmill-labs/windmill/commit/02e4697157112581e9e2a9303180077126db714b)) +* operators get AI sessions ([#11443](https://github.com/windmill-labs/windmill/issues/11443)) ([338c4e9](https://github.com/windmill-labs/windmill/commit/338c4e921dc3185d886c8b040f253285adf20efb)) +* replace the hub_sync script with an opt-in daily server sync ([#11624](https://github.com/windmill-labs/windmill/issues/11624)) ([2b55a8a](https://github.com/windmill-labs/windmill/commit/2b55a8afbc8a86c6cebe97b1cab02c313b986f22)) +* retry transient AI provider failures before output reaches readers ([#11623](https://github.com/windmill-labs/windmill/issues/11623)) ([d3ddef0](https://github.com/windmill-labs/windmill/commit/d3ddef0b7ec4705aa578c4eca24b96e9152485bd)) +* stop offering deno for new scripts, ship it only on full images ([#11648](https://github.com/windmill-labs/windmill/issues/11648)) ([cda7883](https://github.com/windmill-labs/windmill/commit/cda78830fde1b96bd3de2674e23c245b50e3c051)) + + +### Bug Fixes + +* **cli:** count superadmins as able to preserve ownership on push ([#11653](https://github.com/windmill-labs/windmill/issues/11653)) ([70db567](https://github.com/windmill-labs/windmill/commit/70db567a78b1e1bc9e2dcc9ff2139951d778117e)) +* harden app secret validation ([#11655](https://github.com/windmill-labs/windmill/issues/11655)) ([8556a7b](https://github.com/windmill-labs/windmill/commit/8556a7bebe093e0bd61c33d67415fd553f0070ad)) +* open the chat + menu without waiting on the skills and mcp fetches ([#11645](https://github.com/windmill-labs/windmill/issues/11645)) ([1fea1aa](https://github.com/windmill-labs/windmill/commit/1fea1aa6609d644e02bac8d16186f9a2dfcb63cd)) +* say Running on a chat run card only once its job exists ([#11627](https://github.com/windmill-labs/windmill/issues/11627)) ([d046187](https://github.com/windmill-labs/windmill/commit/d046187e3edd01535240c6080b483afe154ae189)) + +## [1.829.0](https://github.com/windmill-labs/windmill/compare/v1.828.0...v1.829.0) (2026-10-09) + + +### Features + +* add an api base url instance setting for displayed urls ([#11628](https://github.com/windmill-labs/windmill/issues/11628)) ([6acf185](https://github.com/windmill-labs/windmill/commit/6acf1858540e83b65b5218c50c98af692092d22e)) +* add Pi template to the AI sandbox picker ([#11619](https://github.com/windmill-labs/windmill/issues/11619)) ([a85d0b7](https://github.com/windmill-labs/windmill/commit/a85d0b76cc494aaa4a484a273ec534bdc2958102)) +* agent nits: multi-select, memory-off notice, agent step label ([#11622](https://github.com/windmill-labs/windmill/issues/11622)) ([c52ba2f](https://github.com/windmill-labs/windmill/commit/c52ba2fd30e19986522420ff02fae224a075b4f6)) +* export worker cpu, memory and occupancy metrics ([#11629](https://github.com/windmill-labs/windmill/issues/11629)) ([5d90f96](https://github.com/windmill-labs/windmill/commit/5d90f96863b64e9bc88ed35429f8068a6a788d1e)) +* let a job see that its worker is draining ([#11612](https://github.com/windmill-labs/windmill/issues/11612)) ([588d846](https://github.com/windmill-labs/windmill/commit/588d8463dfa1c8936b8753b5f0fd794ecb115203)) +* let workspace admins delete other users' drafts ([#11644](https://github.com/windmill-labs/windmill/issues/11644)) ([68376fc](https://github.com/windmill-labs/windmill/commit/68376fc011ef6406a820c69217058316514d3f8d)) +* per-script resource rollup and heaviest scripts table ([#11631](https://github.com/windmill-labs/windmill/issues/11631)) ([442357d](https://github.com/windmill-labs/windmill/commit/442357dbe8db0c52995572acd7385dedf4efdc42)) +* pipeline editor: assets-only view, Fix actions and draft triggers ([#11440](https://github.com/windmill-labs/windmill/issues/11440)) ([3dae553](https://github.com/windmill-labs/windmill/commit/3dae553576ba23b2c1f56e4deac01305c4645c01)) + + +### Bug Fixes + +* advertise the actual signing algorithm in OIDC discovery ([#11637](https://github.com/windmill-labs/windmill/issues/11637)) ([98fc65a](https://github.com/windmill-labs/windmill/commit/98fc65a16bbe2bc9515cdd5443304053c4322274)) +* ask a workspace for its session backups before pushing to it ([#11641](https://github.com/windmill-labs/windmill/issues/11641)) ([2ea846f](https://github.com/windmill-labs/windmill/commit/2ea846f563f5a77cbd3eae2eddee9e38a3700d00)) +* keep instance settings users sub-tabs from being squashed ([#11633](https://github.com/windmill-labs/windmill/issues/11633)) ([fb22e5c](https://github.com/windmill-labs/windmill/commit/fb22e5ce368eba9146e0d51b8fdabb52ca9f68f0)) +* keep raw app deploy panel settings shared with the editor ([#11616](https://github.com/windmill-labs/windmill/issues/11616)) ([0a95ea4](https://github.com/windmill-labs/windmill/commit/0a95ea47facaede2545cc4741d4a70c1c847e11a)) +* load schedules once per page load, poll critical alerts every 30s ([#11639](https://github.com/windmill-labs/windmill/issues/11639)) ([1b4e686](https://github.com/windmill-labs/windmill/commit/1b4e686b2f5de3138df0c3b6ed627ea0fe0a2f7d)) +* name the open workspace settings tab in the page header breadcrumb ([#11617](https://github.com/windmill-labs/windmill/issues/11617)) ([e233908](https://github.com/windmill-labs/windmill/commit/e233908c1e99a246dc7c94fa86ea82c5cbe9d6b3)) +* **python:** add missing import-to-package mappings for common modules ([#11638](https://github.com/windmill-labs/windmill/issues/11638)) ([ab8606f](https://github.com/windmill-labs/windmill/commit/ab8606f37921913ef1410da735abb5742c025225)) +* send hub script completions to the workspace handlers ([#11613](https://github.com/windmill-labs/windmill/issues/11613)) ([d685332](https://github.com/windmill-labs/windmill/commit/d685332771dcdfc718d7a2077fe1cdb3b323909b)) +* set WM_END_USER_EMAIL on direct script and flow runs ([#11636](https://github.com/windmill-labs/windmill/issues/11636)) ([2bdeceb](https://github.com/windmill-labs/windmill/commit/2bdeceba7fb6815cf3150ecb856afb8eb3872e44)) +* store a WAC parent's first start and wall duration ([#11625](https://github.com/windmill-labs/windmill/issues/11625)) ([8f28a58](https://github.com/windmill-labs/windmill/commit/8f28a5860595dbe558e6777a06bb97b6ec6a3e67)) +* sync back volume files rewritten to the same size ([#11620](https://github.com/windmill-labs/windmill/issues/11620)) ([2758920](https://github.com/windmill-labs/windmill/commit/2758920f1182ddd12e6473a644a4a966a435522e)) +* uniform decrypt error and clear stored credentials on logout ([#11632](https://github.com/windmill-labs/windmill/issues/11632)) ([987c321](https://github.com/windmill-labs/windmill/commit/987c321ebcd305df79fa945e941abc2b2e1d3e95)) + +## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08) + + +### Features + +* add gpt-6.1-sol to the chat, agent steps and weekly ai evals ([#11584](https://github.com/windmill-labs/windmill/issues/11584)) ([8f7e0c7](https://github.com/windmill-labs/windmill/commit/8f7e0c72848b56fa97f6f0141d930c7d15044864)) +* add wmill user list and document post-suspend approvers context ([#11600](https://github.com/windmill-labs/windmill/issues/11600)) ([1c2667c](https://github.com/windmill-labs/windmill/commit/1c2667cd5530925893cf69fd622938b6ffa55d1d)) +* ask before minting the frontend sdk token in the raw app editor ([#11604](https://github.com/windmill-labs/windmill/issues/11604)) ([6e948b0](https://github.com/windmill-labs/windmill/commit/6e948b07d13b85e9251c3a200bf149f167649bc4)) +* improve data table roles and grants UX ([#11562](https://github.com/windmill-labs/windmill/issues/11562)) ([d7d18f0](https://github.com/windmill-labs/windmill/commit/d7d18f00af4e2df8fcb1362c9439d2250d0ce4b6)) +* one page header across routes, and one sidebar for operators and developers ([#11310](https://github.com/windmill-labs/windmill/issues/11310)) ([41162b2](https://github.com/windmill-labs/windmill/commit/41162b270c70ddac6a256bde9acf31a629990e24)) + + +### Bug Fixes + +* add retry to runScriptByPathAsync and runScriptByHashAsync ([#11485](https://github.com/windmill-labs/windmill/issues/11485)) ([3f3b1c2](https://github.com/windmill-labs/windmill/commit/3f3b1c2c308ddaba377db45308711e7c660d8775)) +* describe exponential retry as multiplier × seconds^attempt ([#11546](https://github.com/windmill-labs/windmill/issues/11546)) ([a2e51cd](https://github.com/windmill-labs/windmill/commit/a2e51cdbaafb7d71b8e8bbd818e40c472098f9ba)) +* keep capturing job logs after a non-UTF-8 output line ([#11595](https://github.com/windmill-labs/windmill/issues/11595)) ([0e7d974](https://github.com/windmill-labs/windmill/commit/0e7d974ce9b468353ecb6618dace710704690687)) + + +### Performance Improvements + +* load public raw apps without the loading screen ([#11608](https://github.com/windmill-labs/windmill/issues/11608)) ([c09764f](https://github.com/windmill-labs/windmill/commit/c09764fa6516badddbdbb520af32e5d1541cf996)) + +## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07) + + +### Features + +* audit instance settings writes, stable audit OTel target ([#11593](https://github.com/windmill-labs/windmill/issues/11593)) ([0792332](https://github.com/windmill-labs/windmill/commit/0792332859a88b1359ea1be64250d38dd071a0a9)) +* default new raw apps to react 19 and ask for the data setup ([#11586](https://github.com/windmill-labs/windmill/issues/11586)) ([818ddd5](https://github.com/windmill-labs/windmill/commit/818ddd5d31bd81507e274dd18022c1cdbd9be842)) +* export queue delay as OTel and Prometheus metrics ([#11588](https://github.com/windmill-labs/windmill/issues/11588)) ([7565349](https://github.com/windmill-labs/windmill/commit/75653496e542550039cbe5c413d8fcb9d395cea8)) +* pause a worker group without stopping its workers ([#11596](https://github.com/windmill-labs/windmill/issues/11596)) ([95c4b30](https://github.com/windmill-labs/windmill/commit/95c4b3096cf10c1fe59de7a1a6876dfb5f02c39f)) +* run AI decision steps on OpenAI's Decisions API ([#11590](https://github.com/windmill-labs/windmill/issues/11590)) ([046df9e](https://github.com/windmill-labs/windmill/commit/046df9e44db2f2fae3059ccd7a3d6c6955ee8971)) + + +### Bug Fixes + +* avoid URLSearchParams.keys() so the frontend check passes ([#11599](https://github.com/windmill-labs/windmill/issues/11599)) ([fc6c84c](https://github.com/windmill-labs/windmill/commit/fc6c84c6974bba1850e90a088fad5460c8c93ba3)) +* let recover: true from the error handler turn nested failures green ([#11559](https://github.com/windmill-labs/windmill/issues/11559)) ([03e484e](https://github.com/windmill-labs/windmill/commit/03e484e59757c0d4d6f74511d9aa5a1e2f94af67)) +* run dependency resolution and build steps under nsjail ([#11609](https://github.com/windmill-labs/windmill/issues/11609)) ([a238d01](https://github.com/windmill-labs/windmill/commit/a238d01ef6a8c285aef6886c974904f24339f7a2)) +* scope the python wheel cache to workspaces with their own registry ([#11597](https://github.com/windmill-labs/windmill/issues/11597)) ([14381ce](https://github.com/windmill-labs/windmill/commit/14381cefee6fb2b0ece52fbf0001c2ce61bde1a1)) +* type snowflake columns in database manager insert and alter forms ([#11578](https://github.com/windmill-labs/windmill/issues/11578)) ([f2fb156](https://github.com/windmill-labs/windmill/commit/f2fb15603b2e0d11bd5d04dd675212a5e4a7b9d7)) + ## [1.826.0](https://github.com/windmill-labs/windmill/compare/v1.825.0...v1.826.0) (2026-10-07) diff --git a/Dockerfile b/Dockerfile index b1b3f939e7..c6a4ccc55f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,7 +83,6 @@ COPY /python-client/docs/ /frontend/static/pydocs/ RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" -ARG VITE_BASE_URL "" # Must be declared for the build-arg to reach the bundle. See frontend/svelte.config.js. ARG WM_BUILD_VERSION="" # Read more about macro in docker/dev.nu @@ -242,7 +241,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtimes to temp build location (will copy with world-writable perms later) # --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run @@ -287,8 +286,6 @@ COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so ${APP}/libwindmill_duckdb_ffi_internal.so -COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno - COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun # Install windmill CLI diff --git a/README.md b/README.md index 87fc66f399..e9ed6381f1 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can run scripts locally by passing the right environment variables for the ` - **Frontend**: Svelte 5 - **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation - **Runtimes**: - - TypeScript/JavaScript: Bun (default) and Deno + - TypeScript/JavaScript: Bun (default); Deno only on the `windmill-full` images - Python: python3 with uv for dependency management - Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible @@ -318,7 +318,7 @@ running options. This will also avoid compile time issue with sqlx's `query!` macro. 2. (optional, linux only) Install [nsjail](https://github.com/google/nsjail) and have it accessible in your PATH -3. Install bun, deno and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun`,`/usr/bin/deno`, and +3. Install bun and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun` and `/usr/local/bin/python3` or set the corresponding environment variables. 4. (optional) Install the [lld linker](https://lld.llvm.org/) 5. Go to `frontend/`: diff --git a/ai_evals/README.md b/ai_evals/README.md index 44870ba2a2..9b8850c435 100644 --- a/ai_evals/README.md +++ b/ai_evals/README.md @@ -196,7 +196,10 @@ not have read is itself the failure — e.g. `write_variable.value` on a secret Global (and flow) initial fixtures can seed `workspace.datatables` so the `list_datatables`, `get_datatable_table_schema`, and `exec_datatable_sql` tools return seeded data during evals. Each entry is -`{ datatable_name, schemas: { : { : { columns, rows? } } } }`. +`{ datatable_name, schemas: { : {
: { columns, rows? } } } }`, +plus `migrations_enabled: true` and `migrations: [{ timestamp, name, code_up, code_down?, applied? }]` +for a datatable that keeps its schema as migrations, which the migration tools then +read and write in memory. SQL runs through a small in-memory engine (`datatableSqlEngine.ts`), not a real database. Writes are **stateful within a case**: `CREATE`/`DROP`/`INSERT`/`UPDATE`/ `DELETE` mutate the seeded datatable in place, so a later `list_datatables`, diff --git a/ai_evals/adapters/frontend/datatableSqlEngine.ts b/ai_evals/adapters/frontend/datatableSqlEngine.ts index 453eca286e..96a62f6804 100644 --- a/ai_evals/adapters/frontend/datatableSqlEngine.ts +++ b/ai_evals/adapters/frontend/datatableSqlEngine.ts @@ -30,6 +30,17 @@ export interface BenchmarkDatatableSeed { [table: string]: BenchmarkDatatableTableSeed } } + /** Whether the datatable keeps its schema as migrations. */ + migrations_enabled?: boolean + migrations?: BenchmarkDatatableMigrationSeed[] +} + +export interface BenchmarkDatatableMigrationSeed { + timestamp: number + name: string + code_up: string + code_down?: string + applied?: boolean } export interface DatatableSqlResult { diff --git a/ai_evals/adapters/frontend/mockBackend.ts b/ai_evals/adapters/frontend/mockBackend.ts index 2f47e85be6..d8284039b9 100644 --- a/ai_evals/adapters/frontend/mockBackend.ts +++ b/ai_evals/adapters/frontend/mockBackend.ts @@ -692,10 +692,138 @@ export function listBenchmarkDatatables(workspace: string): DataTableTables[] | datatable_name: datatable.datatable_name, schemas: Object.fromEntries( Object.entries(datatable.schemas).map(([schema, tables]) => [schema, Object.keys(tables)]) - ) + ), + ...(datatable.migrations_enabled && { migrations_enabled: true }) })) } +// ============= Datatable migrations ============= + +function benchmarkDatatable(workspace: string, datatableName: string): BenchmarkDatatableSeed { + const datatable = (benchmarkWorkspaceRunnables.get(workspace)?.datatables ?? []).find( + (entry) => entry.datatable_name === datatableName + ) + if (!datatable) { + throw new Error(`datatable "${datatableName}" not found`) + } + return datatable +} + +/** Run a migration body through the in-memory engine, one statement at a time and without the + * transaction frame, so the tables it creates show up in later reads. */ +function applyBenchmarkMigrationSql(datatable: BenchmarkDatatableSeed, sql: string): void { + // Split on the semicolons outside single-quoted literals. + for (const statement of sql.match(/(?:'(?:[^']|'')*'|[^;'])+/g) ?? []) { + const trimmed = statement.replace(/^\s*--.*$/gm, '').trim() + if (trimmed !== '' && !/^(begin|end|commit)$/i.test(trimmed)) { + applyDatatableSql(datatable, trimmed) + } + } +} + +export function getBenchmarkDatatableMigrationsStatus(input: { + workspace: string + datatableName: string +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + if (!datatable.migrations_enabled) { + return { enabled: false, migrations: [] } + } + return { + enabled: true, + migrations: (datatable.migrations ?? []).map(({ applied, ...migration }) => ({ + ...migration, + status: applied ? ('ran' as const) : ('not_run' as const) + })) + } +} + +export function createBenchmarkDatatableMigration(input: { + workspace: string + datatableName: string + requestBody: { name: string; code_up: string; code_down?: string } +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + if (!datatable.migrations_enabled) { + throw new Error( + `Migrations are not enabled for data table '${input.datatableName}'. Enable them first.` + ) + } + const migrations = (datatable.migrations ??= []) + const timestamp = Math.max(20260101000000, ...migrations.map((m) => m.timestamp)) + 1 + migrations.push({ timestamp, ...input.requestBody }) + return { datatable: input.datatableName, timestamp, ...input.requestBody } +} + +export function upsertBenchmarkDatatableMigration(input: { + workspace: string + datatableName: string + requestBody: { timestamp: number; name: string; code_up: string; code_down?: string } +}): string { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + const migrations = (datatable.migrations ??= []) + const existing = migrations.find((m) => m.timestamp === input.requestBody.timestamp) + if (existing?.applied) { + throw new Error( + `Migration ${existing.timestamp} on data table '${input.datatableName}' has already been applied and cannot be modified` + ) + } + if (existing) { + // Replaced whole, as the server does: a down left out is stored as none. + delete existing.code_down + Object.assign(existing, input.requestBody) + } else { + migrations.push({ ...input.requestBody }) + migrations.sort((a, b) => a.timestamp - b.timestamp) + } + return `Upserted migration ${input.requestBody.timestamp} in ${input.datatableName}` +} + +export function runBenchmarkDatatableMigrations(input: { + workspace: string + datatableName: string + upTo?: number + only?: number +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + const applied: { version: number; name: string }[] = [] + for (const migration of datatable.migrations ?? []) { + if ( + migration.applied || + (input.only !== undefined && migration.timestamp !== input.only) || + (input.only === undefined && input.upTo !== undefined && migration.timestamp > input.upTo) + ) { + continue + } + applyBenchmarkMigrationSql(datatable, migration.code_up) + migration.applied = true + applied.push({ version: migration.timestamp, name: migration.name }) + } + return { applied } +} + +export function rollbackBenchmarkDatatableMigration(input: { + workspace: string + datatableName: string + only?: number +}) { + const datatable = benchmarkDatatable(input.workspace, input.datatableName) + const last = (datatable.migrations ?? []) + .filter((m) => m.applied && (input.only === undefined || m.timestamp === input.only)) + .at(-1) + if (!last) { + return { rolled_back: [] } + } + if (last.code_down === undefined) { + throw new Error( + `Cannot roll back migration ${last.timestamp} (${last.name}): it has no down migration` + ) + } + applyBenchmarkMigrationSql(datatable, last.code_down) + last.applied = false + return { rolled_back: [{ version: last.timestamp, name: last.name }] } +} + // ============= DuckLake catalogs and declared metrics ============= /** Seeded DuckLake names, or `null` for a non-benchmark workspace. */ diff --git a/ai_evals/adapters/frontend/vitestAdapter.test.ts b/ai_evals/adapters/frontend/vitestAdapter.test.ts index 288c67e3eb..cb7469b45e 100644 --- a/ai_evals/adapters/frontend/vitestAdapter.test.ts +++ b/ai_evals/adapters/frontend/vitestAdapter.test.ts @@ -92,6 +92,11 @@ vi.mock('$lib/gen', async () => { getBenchmarkCompletedJob, getBenchmarkCompletedJobResultMaybe, getBenchmarkDatatableSchema, + getBenchmarkDatatableMigrationsStatus, + createBenchmarkDatatableMigration, + upsertBenchmarkDatatableMigration, + runBenchmarkDatatableMigrations, + rollbackBenchmarkDatatableMigration, getBenchmarkDraftForUser, getBenchmarkFlowByPath, getBenchmarkFlowAllResults, @@ -400,7 +405,27 @@ vi.mock('$lib/gen', async () => { schemaName: data.schemaName, tableName: data.tableName }) - : actual.WorkspaceService.getDataTableTableSchema(data) + : actual.WorkspaceService.getDataTableTableSchema(data), + getDatatableMigrationsStatus: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? getBenchmarkDatatableMigrationsStatus(data) + : actual.WorkspaceService.getDatatableMigrationsStatus(data), + createDatatableMigration: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? createBenchmarkDatatableMigration(data) + : actual.WorkspaceService.createDatatableMigration(data), + upsertDatatableMigration: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? upsertBenchmarkDatatableMigration(data) + : actual.WorkspaceService.upsertDatatableMigration(data), + runDatatableMigrations: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? runBenchmarkDatatableMigrations(data) + : actual.WorkspaceService.runDatatableMigrations(data), + rollbackDatatableMigrations: async (data: any) => + hasBenchmarkWorkspace(data.workspace) + ? rollbackBenchmarkDatatableMigration(data) + : actual.WorkspaceService.rollbackDatatableMigrations(data) }), DataMetricService: wrapService(actual.DataMetricService, { listDataMetrics: async (data: { workspace: string }) => diff --git a/ai_evals/cases/global.yaml b/ai_evals/cases/global.yaml index b66af3e8dd..4b6704b229 100644 --- a/ai_evals/cases/global.yaml +++ b/ai_evals/cases/global.yaml @@ -901,6 +901,57 @@ - treats creating the table as a normal SQL statement and does not claim a separate registration step is needed - does not write a script to create the table +# The same request on a data table that keeps its schema as migrations: the table is +# created by a migration, since plain SQL would leave it out of the history. +- id: global-datatable-schema-change-uses-migration + prompt: |- + Add a new table called refunds to our workspace data table, with an order id and a refund amount. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json + runtime: + maxTurns: 8 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - create_datatable_migration + forbiddenToolsUsed: + - write_script + - deploy_workspace_item + - delete_workspace_item + toolCallArgs: + - tool: create_datatable_migration + field: code_up + stringIncludesAnyOf: + - create table + skipJudge: true + judgeChecklist: + - notices that the data table keeps its schema as migrations + - creates the refunds table through a migration rather than a plain CREATE TABLE statement + - includes an order id and a refund amount column + - gives the migration a down that drops the table + +# Row data is not schema: it is written with plain SQL even where migrations are enabled. +- id: global-datatable-row-change-skips-migration + prompt: |- + Mark order number 2 as shipped in our workspace data table. + initial: ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json + runtime: + maxTurns: 12 + validate: + draftCountExactly: 0 + toolExpect: + requiredToolsUsed: + - exec_datatable_sql + forbiddenToolsUsed: + - create_datatable_migration + - write_script + - deploy_workspace_item + - delete_workspace_item + skipJudge: true + judgeChecklist: + - runs an UPDATE on the orders table setting the status of order id 2 to shipped + - does not create a migration for a change to row data + - id: global-test25-datatable-mutate-rows prompt: |- Mark order number 2 as shipped in our workspace data table. diff --git a/ai_evals/core/models.ts b/ai_evals/core/models.ts index 22af20d5c3..cf131337fd 100644 --- a/ai_evals/core/models.ts +++ b/ai_evals/core/models.ts @@ -148,6 +148,15 @@ export const EVAL_MODELS: EvalModelSpec[] = [ model: "gpt-6-astra", }, }, + { + id: "gpt-6.1-sol", + label: "GPT-6.1 Sol", + aliases: ["gpt-6.1-sol"], + frontend: { + provider: "openai", + model: "gpt-6.1-sol", + }, + }, { id: "gpt-6-sol", label: "GPT-6 Sol", diff --git a/ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json b/ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json new file mode 100644 index 0000000000..d5227d9af8 --- /dev/null +++ b/ai_evals/fixtures/frontend/global/initial/datatable_orders_migrations_seed.json @@ -0,0 +1,36 @@ +{ + "workspace": { + "datatables": [ + { + "datatable_name": "main", + "migrations_enabled": true, + "migrations": [ + { + "timestamp": 20260501090000, + "name": "create_orders", + "code_up": "BEGIN;\n\nCREATE TABLE orders (id serial PRIMARY KEY, customer_id int, total numeric, status text, created_at timestamptz DEFAULT now());\n\nEND;", + "code_down": "BEGIN;\n\nDROP TABLE orders;\n\nEND;", + "applied": true + } + ], + "schemas": { + "public": { + "orders": { + "columns": { + "id": "int4", + "customer_id": "int4", + "total": "numeric", + "status": "text", + "created_at": "timestamptz" + }, + "rows": [ + { "id": 1, "customer_id": 1, "total": 42.5, "status": "shipped", "created_at": "2026-05-01T10:00:00Z" }, + { "id": 2, "customer_id": 2, "total": 19.99, "status": "pending", "created_at": "2026-05-02T11:30:00Z" } + ] + } + } + } + } + ] + } +} diff --git a/backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json b/backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json similarity index 66% rename from backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json rename to backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json index f3449712c9..c687589db1 100644 --- a/backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json +++ b/backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true)\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n', true)\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde" + "hash": "009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6" } diff --git a/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json b/backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json similarity index 62% rename from backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json rename to backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json index 6e5f8645bf..a0847ae2ca 100644 --- a/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json +++ b/backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730" + "hash": "1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e" } diff --git a/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json new file mode 100644 index 0000000000..1946f55366 --- /dev/null +++ b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int8", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217" +} diff --git a/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json b/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json new file mode 100644 index 0000000000..6ae002d4f7 --- /dev/null +++ b/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats s WHERE s.hour >= $1 AND NOT EXISTS (SELECT 1 FROM workspace w WHERE w.id = s.workspace_id)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8" +} diff --git a/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json new file mode 100644 index 0000000000..036610e08c --- /dev/null +++ b/backend/.sqlx/query-1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker\n WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "1e3407ad2ec46bd965f2aeede0d3cd4a3f9226a5e0c6b15c2580ddd1dba6199b" +} diff --git a/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json b/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json new file mode 100644 index 0000000000..988ff44bb4 --- /dev/null +++ b/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",\n COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\",\n COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",\n COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flows!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "apps!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "schedules!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "triggers!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80" +} diff --git a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json deleted file mode 100644 index 04b23832d9..0000000000 --- a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Bool", - "Varchar", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead" -} diff --git a/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json b/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json new file mode 100644 index 0000000000..eaa7d60dea --- /dev/null +++ b/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO runnable_job_stats\n (hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n SELECT s.hour, s.workspace_id, s.runnable_path, s.worker_group, s.job_count,\n s.total_duration_ms, s.max_memory_peak, s.sum_memory_peak,\n s.memory_sample_count, s.total_cpu_ms\n FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::int[], $6::bigint[],\n $7::int[], $8::bigint[], $9::int[], $10::bigint[])\n AS s(hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n JOIN workspace w ON w.id = s.workspace_id\n ORDER BY s.hour, s.workspace_id, s.runnable_path, s.worker_group\n ON CONFLICT (hour, workspace_id, runnable_path, worker_group) DO UPDATE SET\n job_count = runnable_job_stats.job_count + EXCLUDED.job_count,\n total_duration_ms = runnable_job_stats.total_duration_ms + EXCLUDED.total_duration_ms,\n max_memory_peak = GREATEST(runnable_job_stats.max_memory_peak, EXCLUDED.max_memory_peak),\n sum_memory_peak = runnable_job_stats.sum_memory_peak + EXCLUDED.sum_memory_peak,\n memory_sample_count = runnable_job_stats.memory_sample_count + EXCLUDED.memory_sample_count,\n total_cpu_ms = runnable_job_stats.total_cpu_ms + EXCLUDED.total_cpu_ms\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "TextArray", + "TextArray", + "TextArray", + "Int4Array", + "Int8Array", + "Int4Array", + "Int8Array", + "Int4Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875" +} diff --git a/backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json b/backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json similarity index 69% rename from backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json rename to backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json index 90b559a9b6..37f090c2b4 100644 --- a/backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json +++ b/backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877" + "hash": "344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d" } diff --git a/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json deleted file mode 100644 index 2f794e6be0..0000000000 --- a/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Text", - "Text", - "Bool", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34" -} diff --git a/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json b/backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json similarity index 58% rename from backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json rename to backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json index 532fc430ab..402a406820 100644 --- a/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json +++ b/backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95" + "hash": "42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779" } diff --git a/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json new file mode 100644 index 0000000000..e03d995f15 --- /dev/null +++ b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT canceled_by IS NOT NULL FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0" +} diff --git a/backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json b/backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json similarity index 59% rename from backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json rename to backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json index c747e99795..751b727d0f 100644 --- a/backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json +++ b/backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4" + "hash": "69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927" } diff --git a/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json new file mode 100644 index 0000000000..ba1cc13fac --- /dev/null +++ b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray", + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e" +} diff --git a/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json b/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json deleted file mode 100644 index b647ec29fa..0000000000 --- a/backend/.sqlx/query-71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n id,\n runnable_path,\n permissioned_as_email,\n parent_job\n FROM v2_job\n WHERE workspace_id = 'test-workspace'\n AND permissioned_as_email = 'success_handler@windmill.dev'\n ORDER BY created_at DESC\n LIMIT 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Uuid" - }, - { - "ordinal": 1, - "name": "runnable_path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "permissioned_as_email", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "parent_job", - "type_info": "Uuid" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - true, - false, - true - ] - }, - "hash": "71de038f332593e2177f33284854b6c0bfe9bb4999f2c4e9082a4c5f3b341496" -} diff --git a/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json new file mode 100644 index 0000000000..1c8dfcfa57 --- /dev/null +++ b/backend/.sqlx/query-7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT wp.draining FROM v2_job j\n JOIN v2_job_queue q ON q.id = j.id\n JOIN worker_ping wp ON wp.worker = q.worker\n WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL\n AND ($3::text[] IS NULL OR j.tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "draining", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7963eff5f7109874b94c0940eaef7d4b418a9e82be56dfe7dd65d160e0dcfd13" +} diff --git a/backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json b/backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json similarity index 61% rename from backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json rename to backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json index 6eceb86132..3420b8c337 100644 --- a/backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json +++ b/backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e" + "hash": "8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0" } diff --git a/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json b/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json new file mode 100644 index 0000000000..446e9da6df --- /dev/null +++ b/backend/.sqlx/query-8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, 'true'::jsonb) ON CONFLICT (name) DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "8645e610aa35088376d0919132459bce3ad17df617855e5fcf838f6349e68c97" +} diff --git a/backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json b/backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json similarity index 54% rename from backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json rename to backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json index e103a91c1e..53a9daaf9a 100644 --- a/backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json +++ b/backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2", + "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2", "describe": { "columns": [ { @@ -31,5 +31,5 @@ null ] }, - "hash": "7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec" + "hash": "8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3" } diff --git a/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json new file mode 100644 index 0000000000..ba917c82db --- /dev/null +++ b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683" +} diff --git a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json b/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json deleted file mode 100644 index f105dd7825..0000000000 --- a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d" -} diff --git a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json b/backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json similarity index 57% rename from backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json rename to backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json index bb7e4fa36b..b3a2ea6a05 100644 --- a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json +++ b/backend/.sqlx/query-966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS (SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running)", + "query": "SELECT EXISTS (SELECT 1 FROM global_settings WHERE name = $1)", "describe": { "columns": [ { @@ -11,7 +11,6 @@ ], "parameters": { "Left": [ - "Uuid", "Text" ] }, @@ -19,5 +18,5 @@ null ] }, - "hash": "ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e" + "hash": "966b065bd6bbbff9e0ca15bee504f9cfe5e3a75427767de41e5b85c797f7d58a" } diff --git a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json b/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json deleted file mode 100644 index 0b166b32a4..0000000000 --- a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray", - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5" -} diff --git a/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json b/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json new file mode 100644 index 0000000000..abf8b56d79 --- /dev/null +++ b/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats WHERE hour < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e" +} diff --git a/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json new file mode 100644 index 0000000000..59ed0e1767 --- /dev/null +++ b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d" +} diff --git a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json b/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json deleted file mode 100644 index 2358af807f..0000000000 --- a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665" -} diff --git a/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json new file mode 100644 index 0000000000..e1657e156b --- /dev/null +++ b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c" +} diff --git a/backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json b/backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json similarity index 61% rename from backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json rename to backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json index ec3ae54f17..a5a8c00494 100644 --- a/backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json +++ b/backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c" + "hash": "b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a" } diff --git a/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json b/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json new file mode 100644 index 0000000000..d65f75636d --- /dev/null +++ b/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37" +} diff --git a/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json b/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json new file mode 100644 index 0000000000..be4fbb73c9 --- /dev/null +++ b/backend/.sqlx/query-c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM token WHERE email = $1 AND label = 'session' AND token_hash != $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c3c51c3536eaa6a5e702d306344db63d19f1c9ab0366e01c0092aa653ab51877" +} diff --git a/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json b/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json new file mode 100644 index 0000000000..d84578d37c --- /dev/null +++ b/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT workspace_id, runnable_path, worker_group,\n SUM(job_count)::bigint AS \"job_count!\",\n SUM(total_duration_ms)::bigint AS \"total_duration_ms!\",\n MAX(max_memory_peak) AS \"max_memory_peak!\",\n SUM(sum_memory_peak)::bigint AS \"sum_memory_peak!\",\n SUM(memory_sample_count)::bigint AS \"memory_sample_count!\",\n SUM(total_cpu_ms)::bigint AS \"total_cpu_ms!\"\n FROM runnable_job_stats\n WHERE hour >= $1\n AND ($2::text IS NULL OR worker_group = $2)\n AND ($3::text IS NULL OR workspace_id = $3)\n GROUP BY workspace_id, runnable_path, worker_group\n ORDER BY CASE $4::text\n WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric\n WHEN 'cpu' THEN SUM(total_cpu_ms)\n ELSE SUM(total_duration_ms)\n END DESC, workspace_id, runnable_path, worker_group\n LIMIT $5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "worker_group", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "job_count!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "total_duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "max_memory_peak!", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "sum_memory_peak!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "memory_sample_count!", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "total_cpu_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477" +} diff --git a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json b/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json deleted file mode 100644 index 7f6fe7a81c..0000000000 --- a/backend/.sqlx/query-c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL\n RETURNING jobs_executed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "jobs_executed", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Text", - "TextArray", - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Int8", - "Int8", - "Text", - "Bool" - ] - }, - "nullable": [ - false - ] - }, - "hash": "c748617e060bc41b5922df4b433f20b6971b993e0671b6ccb45db5ef028550bc" -} diff --git a/backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json b/backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json similarity index 69% rename from backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json rename to backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json index a68e5f1253..91cacf4123 100644 --- a/backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json +++ b/backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20" + "hash": "cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74" } diff --git a/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json new file mode 100644 index 0000000000..ba7a89ef73 --- /dev/null +++ b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8" +} diff --git a/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json b/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json new file mode 100644 index 0000000000..55b335fba0 --- /dev/null +++ b/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8" +} diff --git a/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json new file mode 100644 index 0000000000..48963d9ef7 --- /dev/null +++ b/backend/.sqlx/query-de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.id FROM v2_job_queue q\n JOIN native_retry_attempt n ON n.job_id = q.id\n JOIN v2_job r ON r.id = q.id\n JOIN v2_job a ON a.id = r.parent_job\n WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "UuidArray", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "de1adae1306fb9d877ec6f2d52235b3f57d4fa4c27861b51da6bfe8c74abd429" +} diff --git a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json deleted file mode 100644 index ae893b6791..0000000000 --- a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int8", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd" -} diff --git a/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json new file mode 100644 index 0000000000..9ebf9289c2 --- /dev/null +++ b/backend/.sqlx/query-e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE worker_ping SET draining = true WHERE worker = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "e4b9c0e8a46dbfe29bace8d8b12123a98ec175f9d3dc72e40a1cb40e14dec919" +} diff --git a/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json new file mode 100644 index 0000000000..9f51c69616 --- /dev/null +++ b/backend/.sqlx/query-e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker)\n DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining)\n RETURNING jobs_executed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "jobs_executed", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "TextArray", + "Varchar", + "Varchar", + "TextArray", + "Varchar", + "Int8", + "Int8", + "Text", + "Bool", + "Bool" + ] + }, + "nullable": [ + false + ] + }, + "hash": "e7a8c0ee68b8626eb45a7eacdd364360a5d5c783d6ca0a3c4123ea126ed0e60b" +} diff --git a/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json new file mode 100644 index 0000000000..e22953610d --- /dev/null +++ b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323" +} diff --git a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json similarity index 80% rename from backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json rename to backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json index 4ebad828df..1670a5009e 100644 --- a/backend/.sqlx/query-e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258.json +++ b/backend/.sqlx/query-eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1 AND j.parent_job IS NULL\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", + "query": "INSERT INTO job_resolution (job_id, workspace_id, resolved_by, note, automatic)\n SELECT c.id, c.workspace_id, NULL, NULL, true\n FROM v2_job_completed c\n JOIN v2_job j ON j.id = c.id\n WHERE c.status = 'failure'\n AND c.workspace_id = $2\n AND j.flow_step_id IS NULL\n AND j.runnable_id IS NOT DISTINCT FROM $3\n AND EXISTS (\n SELECT 1 FROM v2_job_completed sc\n JOIN v2_job sj ON sj.id = sc.id\n JOIN native_retry_attempt nra ON nra.job_id = sc.id\n WHERE sc.status = 'success'\n AND sc.workspace_id = $2\n AND sj.parent_job = $1\n AND sj.runnable_id IS NOT DISTINCT FROM $3\n )\n AND (\n (j.parent_job = $1\n AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n OR (c.id = $1\n AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id))\n )\n ON CONFLICT (job_id) DO NOTHING", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "e34cb7cd6ff9d80e77a51ae47c3cfb9b0772d5d3d239273fc5cea322cae4e258" + "hash": "eecea4a5ba0de73129bde6ed0c229690c322bc376b1bfc13ed719126f72cabb9" } diff --git a/backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json b/backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json similarity index 58% rename from backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json rename to backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json index a2a87b680d..81505d64a3 100644 --- a/backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json +++ b/backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT COALESCE((ws.operator_settings->>'builder_flows')::boolean, false)) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT (COALESCE((ws.operator_settings->>'builder_flows')::boolean, false) OR COALESCE((ws.operator_settings->>'builder_apps')::boolean, false))) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", "describe": { "columns": [ { @@ -34,5 +34,5 @@ null ] }, - "hash": "cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b" + "hash": "efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b" } diff --git a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json deleted file mode 100644 index 5f86fb97d8..0000000000 --- a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int4", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed" -} diff --git a/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json b/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json new file mode 100644 index 0000000000..0f57f5d951 --- /dev/null +++ b/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3\n AND ($4::text IS NULL OR email = $4)\n RETURNING 1 as \"one!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18" +} diff --git a/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json b/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json new file mode 100644 index 0000000000..c8134447e6 --- /dev/null +++ b/backend/.sqlx/query-f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO global_settings (name, value) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, updated_at = now() WHERE global_settings.value IS DISTINCT FROM EXCLUDED.value", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f633095ebe2fdabd116d687c7f8cfb8cb76159d8db20a42e0fce71146f9b8aaa" +} diff --git a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json similarity index 86% rename from backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json rename to backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json index 583d2bcc6b..5ee5c7571d 100644 --- a/backend/.sqlx/query-68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca.json +++ b/backend/.sqlx/query-fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -107,6 +107,11 @@ "ordinal": 20, "name": "native_mode", "type_info": "Bool" + }, + { + "ordinal": 21, + "name": "draining", + "type_info": "Bool" } ], "parameters": { @@ -138,8 +143,9 @@ true, true, true, + false, false ] }, - "hash": "68bca8f839e47705b11d312ee874eceaa3d1d24d9053ad4aea94b9f8465585ca" + "hash": "fe22c4609d8b2472c0f83fedb92eceec2b6f1ae1a32fcf375161d04419404d9c" } diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 750e0594e1..fbacb857ee 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -214,7 +214,7 @@ Whole classes of defect compile and unit-test clean: - **Stack overflow from a large buffer in an async block.** An array declared across an `.await` is baked into the future's state; once that future is boxed a few layers deep by the job poller, two 16 KB arrays abort the worker *process* (`thread - 'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers + 'tokio-rt-worker' has overflowed its stack`). Heap-allocate read buffers (`vec![0u8; N]`, not `[0u8; N]`). - Deadlocks from draining only one of a child's pipes, missed cancellation or timeout propagation, and anything depending on the real engine's output format. diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 61e705a11c..1bd38513a7 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -247,7 +238,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" dependencies = [ - "object 0.39.1", + "object", ] [[package]] @@ -508,6 +499,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive 0.4.0", + "asn1-rs-impl 0.1.0", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "asn1-rs" version = "0.6.2" @@ -515,7 +522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ "asn1-rs-derive 0.5.1", - "asn1-rs-impl", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -531,7 +538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive 0.6.0", - "asn1-rs-impl", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -540,6 +547,18 @@ dependencies = [ "time", ] +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + [[package]] name = "asn1-rs-derive" version = "0.5.1" @@ -564,6 +583,17 @@ dependencies = [ "synstructure 0.13.2", ] +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "asn1-rs-impl" version = "0.2.0" @@ -920,6 +950,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow 0.14.0", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.6", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -974,11 +1033,23 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ - "aws-lc-sys", + "aws-lc-sys 0.45.0", "untrusted 0.7.1", "zeroize", ] +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-lc-sys" version = "0.45.0" @@ -1349,7 +1420,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", @@ -1492,44 +1563,17 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "rustc_version 0.4.1", + "rustc_version", "tracing", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http 1.5.0", - "http-body 1.1.0", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.3", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core 0.5.6", + "axum-core", "axum-macros", "bytes", "form_urlencoded", @@ -1558,26 +1602,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.5.0", - "http-body 1.1.0", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.6" @@ -1608,12 +1632,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "az" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5eb007b7cacc6c660343e96f650fedf4b5a77512399eb952ca6642cf8d13f7" - [[package]] name = "backon" version = "1.6.0" @@ -1625,27 +1643,18 @@ dependencies = [ "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide 0.8.9", - "object 0.37.3", - "rustc-demangle", - "windows-link 0.2.1", -] - [[package]] name = "base16ct" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base32" version = "0.5.1" @@ -1723,6 +1732,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease 0.2.37", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.119", +] + [[package]] name = "bindgen" version = "0.71.1" @@ -2018,6 +2047,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.3", +] + [[package]] name = "brotli" version = "7.0.0" @@ -2212,6 +2252,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "calendrical_calculations" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5abbd6eeda6885048d357edc66748eea6e0268e3dd11f326fff5bd248d779c26" +dependencies = [ + "core_maths", + "displaydoc", +] + [[package]] name = "candle-core" version = "0.9.2" @@ -2364,6 +2414,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.45" @@ -2490,11 +2552,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" dependencies = [ "const-oid 0.9.6", - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", "x509-cert", ] +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.5" @@ -2534,6 +2602,19 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2562,6 +2643,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-hex" +version = "1.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2700,6 +2793,21 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2751,6 +2859,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "cron" version = "0.17.0" @@ -2842,6 +2956,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -2859,7 +2989,32 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ + "getrandom 0.4.3", "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", ] [[package]] @@ -2899,13 +3054,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", + "subtle", ] [[package]] name = "curl-sys" -version = "0.4.90+curl-8.21.0" +version = "0.4.91+curl-8.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" +checksum = "cd39579123e45bf762716bb893d43886ed1cabe6289455be29a4f1d4e37afb06" dependencies = [ "cc", "libc", @@ -2926,8 +3082,8 @@ dependencies = [ "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", - "fiat-crypto 0.2.9", - "rustc_version 0.4.1", + "fiat-crypto", + "rustc_version", "subtle", "zeroize", ] @@ -3801,7 +3957,7 @@ checksum = "c72e0409b3dbd60a5bf296cbc273a8e36bb3a3aab6abac389f1891e187d5ce14" dependencies = [ "base64 0.22.1", "capacity_builder", - "deno_error 0.7.3", + "deno_error", "deno_media_type", "deno_terminal", "dprint-swc-ext", @@ -3835,37 +3991,29 @@ dependencies = [ "url", ] -[[package]] -name = "deno_console" -version = "0.209.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c0b8a65dcb7b38c22e5969c6454b3cb0839b7dcfb7a4f0d904de871a4c4416" -dependencies = [ - "deno_core", -] - [[package]] name = "deno_core" -version = "0.352.1" +version = "0.412.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf78f3f72ac8e09b18a588bc26a1b3c5ab853e7fb489889b2f5000654d34db5d" +checksum = "580cb2e0d797d75ff3dfc4a8f2b14aa7470f0fa31654a09a83d4cbb42b5eb9b4" dependencies = [ "anyhow", - "az", - "bincode", + "base64 0.22.1", "bit-set", "bit-vec 0.8.0", + "boxed_error", "bytes", - "capacity_builder", "cooked-waker", - "deno_core_icudata", - "deno_error 0.6.1", + "deno_error", "deno_ops", "deno_path_util", "deno_unsync", + "deno_v8", "futures", "indexmap 2.14.2", + "inventory", "libc", + "num-bigint", "parking_lot", "percent-encoding", "pin-project", @@ -3875,28 +4023,25 @@ dependencies = [ "smallvec", "sourcemap", "static_assertions", + "sys_traits", "thiserror 2.0.21", "tokio", "url", - "v8", + "uuid", "wasm_dep_analyzer", + "windows-sys 0.59.0", ] -[[package]] -name = "deno_core_icudata" -version = "0.74.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" - [[package]] name = "deno_crypto" -version = "0.223.0" +version = "0.273.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beb2568806b930b96fb6440ff16463befa73a36de3378986644e7134c9d52e8f" +checksum = "d9931d22328961b9a9c8d3b666a27312c1f3ddcc7ea78d8610ddfbc934ac61a0" dependencies = [ "aes 0.8.3", "aes-gcm", "aes-kw", + "argon2", "aws-lc-rs", "base64 0.22.1", "cbc", @@ -3904,12 +4049,15 @@ dependencies = [ "ctr", "curve25519-dalek", "deno_core", - "deno_error 0.6.1", - "deno_web", + "deno_crypto_provider", + "deno_error", "ecdsa", - "ed448-goldilocks", - "elliptic-curve", + "elliptic-curve 0.13.8", + "fips203", + "fips205", + "hmac 0.12.1", "num-traits", + "ocb3", "once_cell", "p256", "p384", @@ -3920,21 +4068,34 @@ dependencies = [ "serde_bytes", "sha1", "sha2 0.10.9", - "signature", - "spki", + "sha3", + "signature 2.2.0", + "spki 0.7.3", + "subtle", "thiserror 2.0.21", + "tiny-keccak", "tokio", "uuid", "x25519-dalek", ] [[package]] -name = "deno_error" -version = "0.6.1" +name = "deno_crypto_provider" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612ec3fc481fea759141b0c57810889b0a4fb6fee8f10748677bfe492fd30486" +checksum = "fafd673e5a4eefd6d65db358b00ff60488cfa1be6515406b4371935c2e2e6baa" dependencies = [ - "deno_error_macro 0.6.1", + "aws-lc-sys 0.40.0", + "ed448-goldilocks", +] + +[[package]] +name = "deno_error" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfafd2219b29886a71aecbb3449e462deed1b2c474dc5b12f855f0e58c478931" +dependencies = [ + "deno_error_macro", "libc", "serde", "serde_json", @@ -3942,32 +4103,11 @@ dependencies = [ "url", ] -[[package]] -name = "deno_error" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3007d3f1ea92ea503324ae15883aac0c2de2b8cf6fead62203ff6a67161007ab" -dependencies = [ - "deno_error_macro 0.7.3", - "libc", -] - [[package]] name = "deno_error_macro" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380a4224d5d2c3f84da4d764c4326cac62e9a1e3d4960442d29136fc07be863" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "deno_error_macro" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b565e60a9685cdf312c888665b5f8647ac692a7da7e058a5e2268a466da8eaf" +checksum = "1c28ede88783f14cd8aae46ca89f230c226b40e4a81ab06fa52ed72af84beb2f" dependencies = [ "proc-macro2", "quote", @@ -3976,31 +4116,30 @@ dependencies = [ [[package]] name = "deno_features" -version = "0.6.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487773bb24b92f3b88d1c9ef0d4d15888641a2e2ea4d4cdfd6fda12cae26317c" +checksum = "2fef16a40daeb19acfc5345aa5bb367a36ecffef18b7ad1a594f0f01fe7c20c5" dependencies = [ "deno_core", - "serde", - "serde_json", ] [[package]] name = "deno_fetch" -version = "0.233.0" +version = "0.283.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbe158bec955790105ded69978b1245b97c65092f7bf184add344c0e78efffa" +checksum = "20e792f76d45e20cf6f1ad46157bbe1a0a895a6f09ec4d99b0911112be8e05e3" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "data-url", "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_fs", + "deno_io", "deno_path_util", "deno_permissions", "deno_tls", - "dyn-clone", "error_reporter", "h2 0.4.20", "hickory-resolver", @@ -4011,8 +4150,7 @@ dependencies = [ "hyper-util", "ipnet", "percent-encoding", - "rustls-webpki 0.102.8", - "serde", + "rustls-webpki 0.103.15", "serde_json", "thiserror 2.0.21", "tokio", @@ -4021,62 +4159,69 @@ dependencies = [ "tokio-util", "tokio-vsock", "tower 0.5.3", - "tower-http", "tower-service", ] [[package]] name = "deno_fs" -version = "0.119.0" +version = "0.169.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca83a95cea7bcdf19dac1888ef6f04aff69d251640a4d085bc6089c06b4b181" +checksum = "a0b0ae3c55f4af9d33113ba4935828a2e9a7bfcbf4d3058533403b9ef332a966" dependencies = [ "async-trait", - "base32", "boxed_error", "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_io", + "deno_maybe_sync", "deno_path_util", "deno_permissions", "filetime", "junction", "libc", - "nix 0.27.1", + "nix 0.30.1", "rand 0.8.5", "rayon", + "same-file", "serde", "thiserror 2.0.21", - "winapi", + "tokio", "windows-sys 0.59.0", ] [[package]] name = "deno_io" -version = "0.119.0" +version = "0.169.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597aac25be261a1bd545d6f47a80439c60e600c076f9847b3718a592883b2a73" +checksum = "ba495b6c141d34e49fb2dbe7ca746d7ac068b1962470776eb341c59059b758f6" dependencies = [ "async-trait", "deno_core", - "deno_error 0.6.1", + "deno_error", + "deno_permissions", "deno_subprocess_windows", "filetime", - "fs3", "libc", "log", - "nix 0.27.1", - "once_cell", + "nix 0.30.1", "os_pipe", "parking_lot", "pin-project", "rand 0.8.5", "tokio", "uuid", - "winapi", "windows-sys 0.59.0", ] +[[package]] +name = "deno_maybe_sync" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e5e72b38015b5e478b8d739dfa818d563329e03a72d6ce5cfb2ecbcd7f8d41" +dependencies = [ + "rustc-hash 2.1.3", +] + [[package]] name = "deno_media_type" version = "0.3.3" @@ -4103,17 +4248,21 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.201.0" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb88f1ea2762065d6cc0316372acc22696ab6004943aa029f80546f9faf1b491" +checksum = "19ba02152d978f30e2eb5f8d074b343363f53cb1364b5c304b3176da916a06c0" dependencies = [ "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_features", "deno_permissions", + "deno_signals", "deno_tls", + "deno_tunnel", "hickory-proto", "hickory-resolver", + "libc", + "log", "pin-project", "quinn", "rustls-tokio-stream", @@ -4125,32 +4274,33 @@ dependencies = [ "tokio-vsock", "url", "web-transport-proto", + "x509-parser 0.15.1", ] [[package]] name = "deno_ops" -version = "0.228.1" +version = "0.288.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf8dbe5abf37d270bb853c5dfe45fbe3b1b6c453877cc11d7fe84e9862a6dbc" +checksum = "977748624aad80164dbe385c8c907bc0041c2e6849f92528e362e03132c64c52" dependencies = [ "indexmap 2.14.2", - "proc-macro-rules", "proc-macro2", "quote", "stringcase", "strum", "strum_macros", "syn 2.0.119", + "syn-match", "thiserror 2.0.21", ] [[package]] name = "deno_path_util" -version = "0.4.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516f813389095889776b81cc9108ff6f336fd9409b4b12fc0138aea23d2708e1" +checksum = "78c7e98943f0d068928906db0c7bde89de684fa32c6a8018caacc4cee2cdd72b" dependencies = [ - "deno_error 0.6.1", + "deno_error", "percent-encoding", "sys_traits", "thiserror 2.0.21", @@ -4159,77 +4309,100 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.68.0" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baa14d9c3fbba59836ffbaac7f736a2f48d4a5fc209a2103d1e1b898915232a5" +checksum = "f8e9d37215f52e29b5ad9d9065c476168ffb888cd2ad93b0ae153f22304851a0" dependencies = [ "capacity_builder", - "deno_error 0.6.1", + "chrono", + "deno_error", "deno_path_util", "deno_terminal", "deno_unsync", + "dunce", "fqdn", - "ipnetwork", + "ipnet", "libc", "log", - "nix 0.27.1", + "nix 0.30.1", "once_cell", "parking_lot", - "percent-encoding", "serde", "serde_json", "sys_traits", - "temp_deno_which", "thiserror 2.0.21", + "unicode-normalization", "url", - "winapi", + "which 8.0.6", + "windows-sys 0.59.0", +] + +[[package]] +name = "deno_signals" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038ed4b09a6d36c81ac436c4d10076e9e01cc67faf93d701c541887f8eeddf36" +dependencies = [ + "libc", + "signal-hook", + "thiserror 2.0.21", + "tokio", "windows-sys 0.59.0", ] [[package]] name = "deno_subprocess_windows" -version = "0.4.0" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc6b10059f0ccb14c6e0319c5275f0407fb2f9ffe405cd555700561999ea4bf" +checksum = "04d4f451b2f94d2847de4895d39ca5aa264b90af34e72fd79ddc1d0da05a5ea0" dependencies = [ - "fastrand 2.5.0", "futures-channel", "libc", + "rand 0.8.5", "windows-sys 0.59.0", ] [[package]] name = "deno_telemetry" -version = "0.31.0" +version = "0.81.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377581966bd34e85ce230f7f4b1dee7e16c6bf7f845b4690648507ce7ba105d9" +checksum = "8831dc236ed1c66a90c8dfddab93ad44fbb28950fc476c78306b9100233ff694" dependencies = [ "async-trait", "deno_core", - "deno_error 0.6.1", + "deno_error", + "deno_net", + "deno_signals", + "deno_terminal", "deno_tls", + "flate2", "http-body-util", "hyper 1.12.0", "hyper-rustls 0.27.10", "hyper-util", "log", "once_cell", - "opentelemetry 0.27.1", - "opentelemetry-http 0.27.0", - "opentelemetry-otlp 0.27.0", - "opentelemetry-semantic-conventions 0.27.0", - "opentelemetry_sdk 0.27.1", + "opentelemetry 0.32.0", + "opentelemetry-http 0.32.0", + "opentelemetry-otlp 0.32.0", + "opentelemetry-proto 0.32.0", + "opentelemetry-semantic-conventions 0.32.1", + "opentelemetry_sdk 0.32.1", "pin-project", + "prost 0.14.4", "serde", + "sys_traits", "thiserror 2.0.21", "tokio", + "tokio-vsock", + "tower-service", ] [[package]] name = "deno_terminal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f71c27009e0141dedd315f1dfa3ebb0a6ca4acce7c080fac576ea415a465f6" +checksum = "f3ba8041ae7319b3ca6a64c399df4112badcbbe0868b4517637647614bede4be" dependencies = [ "once_cell", "termcolor", @@ -4237,23 +4410,40 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.196.0" +version = "0.246.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7835eb6a8d114703b0293573fbb8885bc2c81e8c35363b9e89391dd3c541fa6" +checksum = "6229f7a00441b2acbf7fae58dd14bded0e835f212653402083430826e640f99d" dependencies = [ "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_native_certs", - "rustls 0.23.35", + "log", + "rustls 0.23.40", "rustls-pemfile 2.2.0", "rustls-tokio-stream", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.15", "serde", "thiserror 2.0.21", "tokio", "webpki-roots 0.26.11", ] +[[package]] +name = "deno_tunnel" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e432929e0c167f2a003b963ee406c714fcd947a87d4f9edc7b3255e87be9881d" +dependencies = [ + "pin-project", + "quinn", + "serde", + "serde_json", + "socket2 0.6.5", + "thiserror 2.0.21", + "tokio", + "tracing", +] + [[package]] name = "deno_unsync" version = "0.4.4" @@ -4266,42 +4456,43 @@ dependencies = [ ] [[package]] -name = "deno_url" -version = "0.209.0" +name = "deno_v8" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cccbb10fb59f29b04161055a65f7fe83364f9e8317d4d2ce3bd7979faf63a87" +checksum = "85cdff08dedf79da6a40f97554f47d59d419ba3df8061b8b31789cd2cbe00433" dependencies = [ - "deno_core", - "deno_error 0.6.1", - "urlpattern", + "v8", + "v8x", ] [[package]] name = "deno_web" -version = "0.240.0" +version = "0.290.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6b8a97cc90b6aaea20fe200cfa7e5b6953dd33ecda0af50d6d360387c33df4" +checksum = "7510967ba4ec695959c1a518748c3b564fd9ca7a8f3eb13bd397707a3ca232da" dependencies = [ "async-trait", - "base64-simd", + "brotli 6.0.0", "bytes", + "cssparser", "deno_core", - "deno_error 0.6.1", - "deno_permissions", + "deno_error", "encoding_rs", "flate2", "futures", "serde", "thiserror 2.0.21", "tokio", + "unicode-normalization", + "urlpattern", "uuid", ] [[package]] name = "deno_webidl" -version = "0.209.0" +version = "0.259.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20bbfac0cf15918f7cbd0b55c366ac20ad65c825f362fa684c3104984254d3b" +checksum = "40af42313723a8d3114c4e5b007abd1cab4400a0b68929794c556b2e792409c7" dependencies = [ "deno_core", ] @@ -4319,6 +4510,30 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs 0.5.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der-parser" version = "9.0.0" @@ -4367,6 +4582,25 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-io" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58cc7f4740088458993d183a200fe56e62378736f94bf0e2dd45807407e7bb94" +dependencies = [ + "derive-io-macros", + "tokio", +] + +[[package]] +name = "derive-io-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d0a1bd7eeab72097740967d03d53db5fbaf8e3c0dd471ebdefa43ce445a20a6" +dependencies = [ + "proc-macro2", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -4456,7 +4690,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "unicode-xid", ] [[package]] @@ -4467,7 +4700,7 @@ checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version", "syn 2.0.119", ] @@ -4524,6 +4757,38 @@ dependencies = [ "ctutils", ] +[[package]] +name = "diplomat" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b37a6cea72eb6df5143d3e091d4c34cb0cf4046dd1ddcc082988886fe1f809" +dependencies = [ + "diplomat_core", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "diplomat-runtime" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae517853d9f82b2d144393b1c32c34c46c2e361de2b6c44a8fdfce23beb211cd" + +[[package]] +name = "diplomat_core" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b18e806f9ecb92df828d080efe47984ae1193958295d4a84fb8b32c9a1aa1b" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "smallvec", + "strck", + "syn 3.0.6", +] + [[package]] name = "dirs" version = "4.0.0" @@ -4681,6 +4946,27 @@ dependencies = [ "text_lines", ] +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "duct" version = "0.13.7" @@ -4727,12 +5013,12 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", - "elliptic-curve", + "elliptic-curve 0.13.8", "rfc6979", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -4741,8 +5027,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -4756,21 +5042,35 @@ dependencies = [ "rand_core 0.6.4", "serde", "sha2 0.10.9", - "signature", + "signature 2.2.0", "subtle", "zeroize", ] [[package]] -name = "ed448-goldilocks" -version = "0.8.3" +name = "ed448" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06924531e9e90130842b012e447f85bdaf9161bc8a0f8092be8cb70b01ebe092" +checksum = "ae112a25f86ae3598d4e8533ed1e65149cec6eb21918e7a6f4c06dddec370263" dependencies = [ - "fiat-crypto 0.1.20", - "hex", + "pkcs8 0.11.0", + "signature 3.0.0", +] + +[[package]] +name = "ed448-goldilocks" +version = "0.14.0-pre.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b805154de2e68f59874ec217ca36790dcffe500cd872c60fe509d28d0814a74d" +dependencies = [ + "ed448", + "elliptic-curve 0.14.1", + "hash2curve", + "rand_core 0.10.1", + "serdect 0.4.3", + "shake", + "signature 3.0.0", "subtle", - "zeroize", ] [[package]] @@ -4800,20 +5100,39 @@ version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", + "base16ct 0.2.0", "base64ct", - "crypto-bigint", + "crypto-bigint 0.5.5", "digest 0.10.7", - "ff", + "ff 0.13.1", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "serde_json", - "serdect", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" +dependencies = [ + "base16ct 1.0.0", + "crypto-bigint 0.7.5", + "crypto-common 0.2.2", + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1 0.8.1", "subtle", "zeroize", ] @@ -5012,6 +5331,18 @@ dependencies = [ "regex-syntax 0.8.11", ] +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash 0.2.0", + "libm", + "portable-atomic", + "siphasher 1.0.4", +] + [[package]] name = "fastdivide" version = "0.4.2" @@ -5044,10 +5375,14 @@ dependencies = [ ] [[package]] -name = "fiat-crypto" -version = "0.1.20" +name = "ff" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] [[package]] name = "fiat-crypto" @@ -5071,6 +5406,30 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "fips203" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8bdb6454f692ca2a2b45cd554c6828c639d7f9c968cf83a678899ec4443a280" +dependencies = [ + "rand_core 0.6.4", + "sha3", + "subtle", + "zeroize", +] + +[[package]] +name = "fips205" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5626bf5534df4ebdbd2536465d7eaa8a9dc2cdeb7e036e0ecf291dcc80ffb6" +dependencies = [ + "rand_core 0.6.4", + "sha2 0.10.9", + "sha3", + "zeroize", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -5090,7 +5449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ "bitflags 2.13.2", - "rustc_version 0.4.1", + "rustc_version", ] [[package]] @@ -5173,9 +5532,9 @@ dependencies = [ [[package]] name = "fqdn" -version = "0.3.12" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb540cf7bc4fe6df9d8f7f0c974cfd0dce8ed4e9e8884e73433b503ee78b4e7d" +checksum = "886ac788f62d16d6b0f26b2fa762b34ef16ebfb4b624c2c15fbcadc9173c0f72" [[package]] name = "from_variant" @@ -5193,17 +5552,6 @@ version = "1.20260821.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c70238056ad1dbfdf035222a571d2e9baf5b6a4c308cfa87bc8cbe7f2f3840" -[[package]] -name = "fs3" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb17cf6ed704f72485332f6ab65257460c4f9f3083934cf402bf9f5b3b600a90" -dependencies = [ - "libc", - "rustc_version 0.2.3", - "winapi", -] - [[package]] name = "fs4" version = "0.13.1" @@ -5533,6 +5881,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str 0.10.0", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -5614,12 +5987,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "git-version" version = "0.3.9" @@ -5721,7 +6088,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "886aa8ec755382a1fdf4651f6e6ec01f2f3bf49f2cb0f068b9a74cafd574a715" dependencies = [ - "prost", + "prost 0.13.5", "prost-types", "tonic 0.12.3", ] @@ -5783,11 +6150,22 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "gzip-header" version = "1.1.0" @@ -5850,6 +6228,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash2curve" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eaf40612d7d854743e7189228a6d528f0f6e8502cf6a0cb831d28a218b7f3f6" +dependencies = [ + "digest 0.11.3", + "elliptic-curve 0.14.1", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -5997,11 +6385,10 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.25.0-alpha.5" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d00147af6310f4392a31680db52a3ed45a2e0f68eb18e8c3fe5537ecc96d9e2" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ - "async-recursion", "async-trait", "cfg-if", "data-encoding", @@ -6013,6 +6400,7 @@ dependencies = [ "ipnet", "once_cell", "rand 0.9.5", + "ring 0.17.14", "serde", "thiserror 2.0.21", "tinyvec", @@ -6023,9 +6411,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.25.0-alpha.5" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5762f69ebdbd4ddb2e975cd24690bf21fe6b2604039189c26acddbc427f12887" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" dependencies = [ "cfg-if", "futures-util", @@ -6217,7 +6605,9 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ + "subtle", "typenum", + "zeroize", ] [[package]] @@ -6344,7 +6734,7 @@ dependencies = [ "hyper 1.12.0", "hyper-util", "log", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "tokio", "tokio-rustls 0.26.6", @@ -6461,6 +6851,28 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_calendar" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58655df2f728e46e4eee80bddebefacf52ec3e77cdb925014b47c5a5905eb55c" +dependencies = [ + "calendrical_calculations", + "displaydoc", + "icu_calendar_data", + "icu_locale_core", + "icu_locale_fallback", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_calendar_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc00caaa3fb3201ff7a18aa458e8c3ab042b9da154cfdf948bdde3936c31c36e" + [[package]] name = "icu_collections" version = "2.3.0" @@ -6684,10 +7096,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] -name = "inventory" -version = "0.3.24" +name = "interpolator" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f0c30c76f2f4ccee3fe55a2435f691ca00c0e4bd87abe4f4a851b1d4dac39b" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" dependencies = [ "rustversion", ] @@ -6703,17 +7121,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "io-uring" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "libc", -] - [[package]] name = "ipconfig" version = "0.3.4" @@ -6733,15 +7140,6 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" -[[package]] -name = "ipnetwork" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" -dependencies = [ - "serde", -] - [[package]] name = "is-macro" version = "0.3.8" @@ -6802,12 +7200,27 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "ixdtf" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3667095d64c3ecffc96463a21157b04bf3e252f6e8d5750b20c02e33c194e3" + [[package]] name = "jiff" version = "0.2.38" @@ -6887,7 +7300,7 @@ checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version", "simd_cesu8", "syn 2.0.119", ] @@ -6995,7 +7408,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature", + "signature 2.2.0", "simple_asn1", "zeroize", ] @@ -7022,6 +7435,25 @@ dependencies = [ "serde_json", ] +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "keccak" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", +] + [[package]] name = "keyed_priority_queue" version = "0.4.2" @@ -7093,7 +7525,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem", - "rustls 0.23.35", + "rustls 0.23.40", "secrecy", "serde", "serde_json", @@ -7166,12 +7598,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lapin" version = "2.5.5" @@ -7582,7 +8008,7 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "smtp-proto", "tokio", @@ -7590,76 +8016,30 @@ dependencies = [ "webpki-roots 0.26.11", ] -[[package]] -name = "malachite" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" -dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", -] - -[[package]] -name = "malachite-base" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" -dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more 1.0.0", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", -] - [[package]] name = "manyhow" version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" dependencies = [ - "manyhow-macros", + "manyhow-macros 0.11.4", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "manyhow" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" +dependencies = [ + "manyhow-macros 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "manyhow-macros" version = "0.11.4" @@ -7671,6 +8051,17 @@ dependencies = [ "quote", ] +[[package]] +name = "manyhow-macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "mappable-rc" version = "0.1.1" @@ -7938,7 +8329,7 @@ checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666" dependencies = [ "darling 0.23.0", "heck", - "manyhow", + "manyhow 0.11.4", "num-bigint", "proc-macro-crate", "proc-macro2", @@ -8072,6 +8463,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] @@ -8291,7 +8683,6 @@ checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", - "rand 0.8.5", "serde", ] @@ -8435,15 +8826,6 @@ dependencies = [ "url", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "object" version = "0.39.1" @@ -8490,6 +8872,18 @@ dependencies = [ "web-time", ] +[[package]] +name = "ocb3" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c196e0276c471c843dd5777e7543a36a298a4be942a2a688d8111cd43390dedb" +dependencies = [ + "aead", + "cipher 0.4.4", + "ctr", + "subtle", +] + [[package]] name = "odpic-sys" version = "0.1.1" @@ -8499,6 +8893,15 @@ dependencies = [ "cc", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs 0.5.2", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -8528,6 +8931,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -8667,23 +9074,23 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.27.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 1.0.69", + "thiserror 2.0.21", "tracing", ] [[package]] name = "opentelemetry" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -8705,18 +9112,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "opentelemetry-http" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" -dependencies = [ - "async-trait", - "bytes", - "http 1.5.0", - "opentelemetry 0.27.1", -] - [[package]] name = "opentelemetry-http" version = "0.30.0" @@ -8731,24 +9126,15 @@ dependencies = [ ] [[package]] -name = "opentelemetry-otlp" -version = "0.27.0" +name = "opentelemetry-http" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", - "futures-core", + "bytes", "http 1.5.0", - "opentelemetry 0.27.1", - "opentelemetry-http 0.27.0", - "opentelemetry-proto 0.27.0", - "opentelemetry_sdk 0.27.1", - "prost", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tonic 0.12.3", - "tracing", + "opentelemetry 0.32.0", ] [[package]] @@ -8762,7 +9148,7 @@ dependencies = [ "opentelemetry-http 0.30.0", "opentelemetry-proto 0.30.0", "opentelemetry_sdk 0.30.0", - "prost", + "prost 0.13.5", "reqwest 0.12.28", "thiserror 2.0.21", "tokio", @@ -8771,17 +9157,19 @@ dependencies = [ ] [[package]] -name = "opentelemetry-proto" -version = "0.27.0" +name = "opentelemetry-otlp" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ - "hex", - "opentelemetry 0.27.1", - "opentelemetry_sdk 0.27.1", - "prost", - "serde", - "tonic 0.12.3", + "http 1.5.0", + "opentelemetry 0.32.0", + "opentelemetry-http 0.32.0", + "opentelemetry-proto 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost 0.14.4", + "serde_json", + "thiserror 2.0.21", ] [[package]] @@ -8794,16 +9182,24 @@ dependencies = [ "hex", "opentelemetry 0.30.0", "opentelemetry_sdk 0.30.0", - "prost", + "prost 0.13.5", "serde", "tonic 0.13.1", ] [[package]] -name = "opentelemetry-semantic-conventions" -version = "0.27.0" +name = "opentelemetry-proto" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost 0.14.4", + "serde", +] [[package]] name = "opentelemetry-semantic-conventions" @@ -8812,25 +9208,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83d059a296a47436748557a353c5e6c5705b9470ef6c95cfc52c21a8814ddac2" [[package]] -name = "opentelemetry_sdk" -version = "0.27.1" +name = "opentelemetry-semantic-conventions" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" -dependencies = [ - "async-trait", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "opentelemetry 0.27.1", - "percent-encoding", - "rand 0.8.5", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tracing", -] +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" [[package]] name = "opentelemetry_sdk" @@ -8850,6 +9231,24 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.21", + "tokio", + "tokio-stream", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -8901,6 +9300,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap 2.14.2", +] + [[package]] name = "os_pipe" version = "1.2.1" @@ -8939,7 +9347,7 @@ checksum = "3cae83056e7cb770211494a0ecf66d9fa7eba7d00977e5bb91f0e925b40b937f" dependencies = [ "cbc", "cms", - "der", + "der 0.7.10", "des 0.8.1", "hex", "hmac 0.12.1", @@ -8960,7 +9368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.9", ] @@ -8972,7 +9380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.9", ] @@ -8983,9 +9391,9 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" dependencies = [ - "base16ct", + "base16ct 0.2.0", "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "rand_core 0.6.4", "sha2 0.10.9", @@ -9224,7 +9632,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_macros", + "phf_macros 0.11.3", "phf_shared 0.11.3", ] @@ -9243,6 +9651,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ + "phf_macros 0.13.1", "phf_shared 0.13.1", "serde", ] @@ -9253,7 +9662,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", + "phf_generator 0.11.3", "phf_shared 0.11.3", ] @@ -9267,19 +9676,42 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand 2.5.0", + "phf_shared 0.13.1", +] + [[package]] name = "phf_macros" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ - "phf_generator", + "phf_generator 0.11.3", "phf_shared 0.11.3", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -9380,9 +9812,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", ] [[package]] @@ -9393,9 +9825,9 @@ checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" dependencies = [ "cms", "const-oid 0.9.6", - "der", + "der 0.7.10", "digest 0.10.7", - "spki", + "spki 0.7.3", "x509-cert", "zeroize", ] @@ -9408,11 +9840,11 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ "aes 0.8.3", "cbc", - "der", + "der 0.7.10", "pbkdf2", "scrypt", "sha2 0.10.9", - "spki", + "spki 0.7.3", ] [[package]] @@ -9421,8 +9853,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.1", ] [[package]] @@ -9608,7 +10050,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -9617,7 +10059,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.15+spec-1.1.0", + "toml_edit 0.25.17+spec-1.1.0", ] [[package]] @@ -9643,29 +10085,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-rules" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" -dependencies = [ - "proc-macro-rules-macros", - "proc-macro2", - "syn 2.0.119", -] - -[[package]] -name = "proc-macro-rules-macros" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "207fffb0fe655d1d47f6af98cc2793405e85929bdbc420d685554ff07be27ac7" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "proc-macro-utils" version = "0.10.0" @@ -9739,6 +10158,21 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bitflags 2.13.2", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax 0.8.11", + "unarray", +] + [[package]] name = "prost" version = "0.13.5" @@ -9746,7 +10180,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive 0.14.4", ] [[package]] @@ -9762,13 +10206,26 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "prost-types" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.13.5", ] [[package]] @@ -9886,7 +10343,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.3", - "rustls 0.23.35", + "rustls 0.23.40", "socket2 0.6.5", "thiserror 2.0.21", "tokio", @@ -9902,13 +10359,14 @@ checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", + "fastbloom", "getrandom 0.4.3", "lru-slab", "rand 0.10.3", "rand_pcg", "ring 0.17.14", "rustc-hash 2.1.3", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "slab", "thiserror 2.0.21", @@ -9940,6 +10398,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -10094,6 +10574,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -10373,7 +10862,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pki-types", "serde", @@ -10420,7 +10909,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -10476,6 +10965,16 @@ dependencies = [ "wasmtimer", ] +[[package]] +name = "resb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22d392791f3c6802a1905a509e9d1a6039cbbcb5e9e00e5a6d3661f7c874f390" +dependencies = [ + "potential_utf", + "serde_core", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -10666,14 +11165,114 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags 2.13.2", + "char_str", + "compact_str 0.10.0", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "thin-vec", + "thiserror 2.0.21", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags 2.13.2", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash 2.1.3", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.5", +] + [[package]] name = "rumqttc" version = "0.24.0" @@ -10747,12 +11346,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "rustc-demangle" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -10765,22 +11358,13 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.28", + "semver", ] [[package]] @@ -10860,9 +11444,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -10881,7 +11465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70cc376c6ba1823ae229bacf8ad93c136d93524eab0e4e5e0e4f96b9c4e5b212" dependencies = [ "log", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.7.3", "rustls-pki-types", "rustls-webpki 0.103.15", @@ -10963,7 +11547,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-platform-verifier-android", "rustls-webpki 0.103.15", @@ -10981,12 +11565,13 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-tokio-stream" -version = "0.5.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa7dc7c991d9164e55bbf1558029eb5b84d32cc4d61a7df5b8641b2deedc4b3" +checksum = "0560d12c0d8c672f849197de91b9ee61f5bf9aa024c97aaeeb112ec2f6c347fd" dependencies = [ + "derive-io", "futures", - "rustls 0.23.35", + "rustls 0.23.40", "socket2 0.5.10", "tokio", ] @@ -11024,63 +11609,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.23" @@ -11288,11 +11816,25 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", - "serdect", + "pkcs8 0.10.2", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct 1.0.0", + "ctutils", + "der 0.8.2", + "hybrid-array", "subtle", "zeroize", ] @@ -11342,27 +11884,12 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "seq-macro" version = "0.3.6" @@ -11541,16 +12068,16 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.261.1" +version = "0.321.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3495190857461e87a2716141043218aad5281f219f54a03b7ebbe605b3b931df" +checksum = "fd3b6f6140192f569b88997e2ea7afeef6aa38b64374e84784ea1d1ea35679d3" dependencies = [ - "deno_error 0.6.1", + "deno_error", + "deno_v8", "num-bigint", "serde", "smallvec", "thiserror 2.0.21", - "v8", ] [[package]] @@ -11620,7 +12147,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" dependencies = [ - "base16ct", + "base16ct 0.2.0", + "serde", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct 1.0.0", "serde", ] @@ -11695,6 +12232,27 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest 0.10.7", + "keccak 0.1.6", +] + +[[package]] +name = "shake" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.2", + "sponge-cursor", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -11735,6 +12293,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -11751,9 +12319,9 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", + "signature 2.2.0", "zeroize", ] @@ -11767,6 +12335,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -11779,7 +12357,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ - "rustc_version 0.4.1", + "rustc_version", "simdutf8", ] @@ -11947,7 +12525,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -11962,6 +12550,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + [[package]] name = "sql-builder" version = "3.1.1" @@ -12041,7 +12635,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_json", "sha2 0.10.9", @@ -12238,6 +12832,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strck" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42316e70da376f3d113a68d138a60d8a9883c604fe97942721ec2068dab13a9f" +dependencies = [ + "unicode-ident", +] + [[package]] name = "string_enum" version = "1.0.2" @@ -12438,7 +13041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcf55c2d7555c93f4945e29f93b7529562be97ba16e60dd94c25724d746174ac" dependencies = [ "ascii", - "compact_str", + "compact_str 0.7.1", "memchr", "num-bigint", "once_cell", @@ -12751,6 +13354,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn-match" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b8f0a9004d6aafa6a588602a1119e6cdaacec9921aa1605383e6e7d6258fd6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -12760,6 +13374,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -12793,11 +13419,14 @@ dependencies = [ [[package]] name = "sys_traits" -version = "0.1.16" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4707edf3196e8037ee45018d1bb1bfb233b0e4fc440fa3d3f25bc69bfdaf26" +checksum = "c7a35f11f9773f11930a7d1e6276766f1577400ee2530b7f0cf3844c2dcbf12b" dependencies = [ + "junction", + "libc", "sys_traits_macros", + "windows-sys 0.59.0", ] [[package]] @@ -13052,15 +13681,6 @@ dependencies = [ "rustls-pemfile 2.2.0", ] -[[package]] -name = "temp_deno_which" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366c5ccd670145885feb6efd6bbf2478ed236c4c3839046fcc8e2a1a84c51091" -dependencies = [ - "either", -] - [[package]] name = "tempfile" version = "3.27.0" @@ -13074,6 +13694,40 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "temporal_capi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f48eb9ca4ca22caaa48d1e7a534ad3fd598a23d1a71945e8ad5ac0be501fa5f2" +dependencies = [ + "diplomat", + "diplomat-runtime", + "icu_calendar", + "icu_locale_core", + "num-traits", + "temporal_rs", + "timezone_provider", + "writeable", + "zoneinfo64", +] + +[[package]] +name = "temporal_rs" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a978fef907a27736827d336c0bfaab14fb8a7509dd6dc2b43c44e51d59aca5b" +dependencies = [ + "calendrical_calculations", + "core_maths", + "icu_calendar", + "icu_locale_core", + "ixdtf", + "num-traits", + "timezone_provider", + "tinystr", + "writeable", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -13112,6 +13766,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -13274,6 +13934,18 @@ dependencies = [ "time-core", ] +[[package]] +name = "timezone_provider" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ec4f4eebb4817fde02a411aedc7ac5f66c89c68c49e5d4b17a8139f077af52" +dependencies = [ + "tinystr", + "zerotrie", + "zerovec", + "zoneinfo64", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -13350,23 +14022,20 @@ dependencies = [ [[package]] name = "tokio" -version = "1.46.1" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.5.10", + "socket2 0.6.5", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13395,13 +14064,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -13490,7 +14159,7 @@ version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.40", "tokio", ] @@ -13622,9 +14291,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ "serde_core", ] @@ -13644,21 +14313,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.15+spec-1.1.0" +version = "0.25.17+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" dependencies = [ "indexmap 2.14.2", - "toml_datetime 1.1.1+spec-1.1.0", + "toml_datetime 1.1.2+spec-1.1.0", "toml_parser", "winnow 1.0.4", ] [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow 1.0.4", ] @@ -13669,13 +14338,10 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", "base64 0.22.1", "bytes", "flate2", - "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -13684,9 +14350,8 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-pemfile 2.2.0", - "socket2 0.5.10", "tokio", "tokio-rustls 0.26.6", "tokio-stream", @@ -13704,7 +14369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "flate2", @@ -13717,7 +14382,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-native-certs 0.8.4", "socket2 0.5.10", "tokio", @@ -13775,7 +14440,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "151b5a3e3c45df17466454bb74e9ecedecc955269bdedbf4d150dfa393b55a36" dependencies = [ - "axum-core 0.5.6", + "axum-core", "cookie", "futures-util", "http 1.5.0", @@ -13973,9 +14638,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" +checksum = "d0af592be68c579aa78a16846bd19422978c3c52e438523d45ff5d1bff1f9d4a" [[package]] name = "tree-sitter-r" @@ -14129,6 +14794,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unic-char-property" version = "0.9.0" @@ -14150,17 +14821,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -14173,17 +14833,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -14271,28 +14920,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unicode_names2" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" -dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.5", -] - [[package]] name = "universal-hash" version = "0.5.1" @@ -14344,7 +14971,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "serde", "serde_json", @@ -14374,13 +15001,13 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "urlpattern" -version = "0.3.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70acd30e3aa1450bc2eece896ce2ad0d178e9c079493819301573dae3c37ba6d" +checksum = "0f805818f843b548bacc19609eb3619dd2850e54746f5cada37927393c2ef4ec" dependencies = [ + "icu_properties", "regex", "serde", - "unic-ucd-ident", "url", ] @@ -14428,20 +15055,36 @@ dependencies = [ [[package]] name = "v8" -version = "137.3.0" +version = "150.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" +checksum = "42a978ff11f15b24e5c05a7123cf2b68f41e763546699781a924ef4e2cf43a49" dependencies = [ - "bindgen 0.71.1", + "bindgen 0.72.1", "bitflags 2.13.2", "fslock", "gzip-header", "home", "miniz_oxide 0.8.9", "paste", + "temporal_capi", "which 6.0.3", ] +[[package]] +name = "v8x" +version = "149.4.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3701ffb3692a24808503d17f758b0a3f688d2c451779247982566b5941033e63" +dependencies = [ + "bindgen 0.70.1", + "bitflags 2.13.2", + "cc", + "paste", + "temporal_capi", + "temporal_rs", + "timezone_provider", +] + [[package]] name = "valuable" version = "0.1.1" @@ -14661,11 +15304,11 @@ dependencies = [ [[package]] name = "wasm_dep_analyzer" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51cf5f08b357e64cd7642ab4bbeb11aecab9e15520692129624fb9908b8df2c" +checksum = "a10e6b67c951a84de7029487e0e0a496860dae49f6699edd279d5ff35b8fbf54" dependencies = [ - "deno_error 0.6.1", + "deno_error", "thiserror 2.0.21", ] @@ -14766,6 +15409,12 @@ dependencies = [ "winsafe", ] +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" + [[package]] name = "whoami" version = "1.6.1" @@ -14829,14 +15478,14 @@ dependencies = [ [[package]] name = "windmill" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "constant_time_eq 0.3.1", @@ -14856,7 +15505,7 @@ dependencies = [ "rdkafka", "reqwest 0.13.5", "rumqttc", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_derive", "serde_json", @@ -14887,6 +15536,7 @@ dependencies = [ "windmill-api-scripts", "windmill-api-settings", "windmill-api-users", + "windmill-audit", "windmill-autoscaling", "windmill-common", "windmill-dep-map", @@ -14918,7 +15568,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.826.0" +version = "1.830.0" dependencies = [ "async-stream", "async-trait", @@ -14935,6 +15585,7 @@ dependencies = [ "indexmap 2.14.2", "lazy_static", "mime_guess", + "rand 0.9.5", "reqwest 0.13.5", "serde", "serde_json", @@ -14952,9 +15603,9 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -14965,7 +15616,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "argon2", @@ -14976,7 +15627,7 @@ dependencies = [ "async_zip", "aws-sdk-config", "aws-sigv4", - "axum 0.8.9", + "axum", "base32", "base64 0.22.1", "bytes", @@ -15008,17 +15659,19 @@ dependencies = [ "native-tls", "openidconnect", "openssl", + "p256", "pin-project", "postgres-native-tls 0.5.3", "prometheus", "quick-xml", "quick_cache", "rand 0.9.5", + "rcgen", "regex", "reqwest 0.13.5", "rsa", "rust-embed", - "rustls 0.23.35", + "rustls 0.23.40", "samael", "serde", "serde_json", @@ -15034,6 +15687,7 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-postgres", + "tokio-rustls 0.26.6", "tokio-stream", "tokio-util", "tower 0.5.3", @@ -15105,9 +15759,9 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "hyper 1.12.0", @@ -15128,9 +15782,9 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15145,10 +15799,10 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "itertools 0.14.0", @@ -15171,7 +15825,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.826.0" +version = "1.830.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15181,9 +15835,9 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "itertools 0.14.0", "serde", @@ -15198,9 +15852,9 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "ed25519-dalek", @@ -15220,10 +15874,10 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "candle-core", "candle-nn", "candle-transformers", @@ -15243,9 +15897,9 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "sql-builder", @@ -15259,9 +15913,9 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "hyper 1.12.0", "serde", @@ -15281,9 +15935,9 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "globset", "lazy_static", @@ -15303,9 +15957,9 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15317,14 +15971,14 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "hex", @@ -15353,10 +16007,10 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "http 1.5.0", @@ -15379,9 +16033,9 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "flate2", "futures", @@ -15389,7 +16043,7 @@ dependencies = [ "lazy_static", "quick_cache", "reqwest 0.13.5", - "semver 1.0.28", + "semver", "serde", "serde_json", "sha2 0.10.9", @@ -15407,10 +16061,10 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "http 1.5.0", "indexmap 2.14.2", "itertools 0.14.0", @@ -15429,9 +16083,9 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "chrono-tz", "serde", @@ -15449,10 +16103,10 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "chrono", "futures", "http 1.5.0", @@ -15491,10 +16145,10 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -15520,7 +16174,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.826.0" +version = "1.830.0" dependencies = [ "lazy_static", "serde", @@ -15532,10 +16186,10 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.826.0" +version = "1.830.0" dependencies = [ "argon2", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "hyper 1.12.0", @@ -15556,9 +16210,9 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15570,9 +16224,9 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.826.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "constant_time_eq 0.3.1", "futures", @@ -15609,7 +16263,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.826.0" +version = "1.830.0" dependencies = [ "chrono", "lazy_static", @@ -15624,10 +16278,10 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "k8s-openapi", "kube", "serde", @@ -15643,7 +16297,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.826.0" +version = "1.830.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15657,7 +16311,7 @@ dependencies = [ "aws-sdk-secretsmanager", "aws-sdk-sts", "aws-smithy-types-convert", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "bitflags 2.13.2", @@ -15679,6 +16333,7 @@ dependencies = [ "git-version", "globset", "hex", + "hkdf", "hmac 0.12.1", "hyper 1.12.0", "indexmap 2.14.2", @@ -15696,6 +16351,7 @@ dependencies = [ "opentelemetry-otlp 0.30.0", "opentelemetry-semantic-conventions 0.30.0", "opentelemetry_sdk 0.30.0", + "p256", "pep440_rs", "phf 0.11.3", "pin-project-lite", @@ -15711,18 +16367,18 @@ dependencies = [ "reqwest-middleware", "reqwest-retry", "rsa", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "ryu-js", "schemars 0.8.22", - "semver 1.0.28", + "semver", "serde", "serde_json", "serde_yml", "sha1", "sha2 0.10.9", "size", - "spki", + "spki 0.7.3", "sqlx", "strum", "strum_macros", @@ -15757,7 +16413,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.826.0" +version = "1.830.0" dependencies = [ "chrono", "futures", @@ -15777,7 +16433,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.826.0" +version = "1.830.0" dependencies = [ "regex", "serde", @@ -15794,7 +16450,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15821,7 +16477,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "futures", @@ -15838,7 +16494,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.826.0" +version = "1.830.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15854,7 +16510,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", @@ -15875,11 +16531,11 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "chrono", @@ -15906,12 +16562,12 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "arc-swap", "async-oauth2", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "hex", @@ -15931,7 +16587,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-stream", @@ -15940,7 +16596,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-sts", "aws-smithy-types-convert", - "axum 0.8.9", + "axum", "bytes", "chrono", "datafusion", @@ -15967,7 +16623,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "futures", @@ -15980,12 +16636,13 @@ dependencies = [ "thiserror 2.0.21", "tokio", "tracing", + "windmill-audit", "windmill-common", ] [[package]] name = "windmill-parser" -version = "1.826.0" +version = "1.830.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15994,7 +16651,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16006,7 +16663,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16018,7 +16675,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "gosyn", @@ -16030,7 +16687,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16042,7 +16699,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16054,7 +16711,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "nu-parser", @@ -16065,7 +16722,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16076,30 +16733,30 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] [[package]] name = "windmill-parser-py-asset" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] [[package]] name = "windmill-parser-py-imports" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", @@ -16109,7 +16766,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -16121,7 +16779,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16133,7 +16791,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16147,7 +16805,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16164,7 +16822,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16177,7 +16835,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde", @@ -16189,7 +16847,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16207,7 +16865,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16223,11 +16881,12 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9", @@ -16239,7 +16898,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16253,11 +16912,11 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.8.9", + "axum", "backon", "chrono", "chrono-tz", @@ -16292,23 +16951,19 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "const_format", "deno_ast", - "deno_console", "deno_core", "deno_crypto", - "deno_error 0.6.1", + "deno_error", "deno_fetch", - "deno_fs", - "deno_io", "deno_net", "deno_permissions", "deno_telemetry", "deno_tls", - "deno_url", "deno_web", "deno_webidl", "futures", @@ -16317,13 +16972,15 @@ dependencies = [ "rcgen", "regex", "reqwest 0.13.5", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_json", "sqlx", + "sys_traits", "tokio", "tracing", "uuid", + "v8", "winapi", "windmill-common", "windmill-parser-ts", @@ -16332,7 +16989,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.826.0" +version = "1.830.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16343,11 +17000,11 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "dashmap", @@ -16378,11 +17035,11 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "chrono", "futures", "serde", @@ -16402,11 +17059,11 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "hyper 1.12.0", @@ -16435,11 +17092,11 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "itertools 0.14.0", @@ -16462,11 +17119,11 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -16495,11 +17152,11 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "lazy_static", "regex", @@ -16515,11 +17172,11 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -16549,11 +17206,11 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "constant_time_eq 0.3.1", @@ -16586,11 +17243,11 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "itertools 0.14.0", "rdkafka", @@ -16609,11 +17266,11 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "itertools 0.14.0", @@ -16633,12 +17290,12 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "itertools 0.14.0", "nkeys", @@ -16657,11 +17314,11 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "byteorder", "bytes", "chrono", @@ -16692,7 +17349,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", @@ -16701,7 +17358,7 @@ dependencies = [ "aws-sdk-sqs", "aws-sdk-sts", "aws-smithy-types", - "axum 0.8.9", + "axum", "backon", "chrono", "itertools 0.14.0", @@ -16720,11 +17377,11 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "http 1.5.0", @@ -16745,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16764,14 +17421,14 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-once-cell", "async-recursion", "async-stream", "async-trait", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -16816,7 +17473,7 @@ dependencies = [ "postgres-protocol", "process-wrap", "prometheus", - "prost", + "prost 0.13.5", "quick_cache", "rand 0.9.5", "rcgen", @@ -16825,7 +17482,7 @@ dependencies = [ "reqwest-middleware", "rsa", "rust_decimal", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pemfile 2.2.0", "serde", @@ -16883,7 +17540,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.826.0" +version = "1.830.0" dependencies = [ "bytes", "futures", @@ -17479,8 +18136,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ "const-oid 0.9.6", - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs 0.5.2", + "data-encoding", + "der-parser 8.2.0", + "lazy_static", + "nom", + "oid-registry 0.6.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", ] [[package]] @@ -17592,18 +18266,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.61" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879fb705ce98c32e41ebdb970fbe1204f8492423b314c6ab0354c3e7b5542866" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.61" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708882a28301d604fa039cc7727607a98d04c4b86dc76ec9cc683f805d709759" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ "proc-macro2", "quote", @@ -17710,6 +18384,19 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zoneinfo64" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6eb2607e906160c457fd573e9297e65029669906b9ac8fb1b5cd5e055f0705" +dependencies = [ + "calendrical_calculations", + "icu_locale_core", + "potential_utf", + "resb", + "serde", +] + [[package]] name = "zstd" version = "0.13.3" @@ -17730,9 +18417,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.1+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" dependencies = [ "cc", "pkg-config", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7808c0d621..aaf5a2e8df 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.826.0" +version = "1.830.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.826.0" +version = "1.830.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -156,7 +156,7 @@ bedrock = ["windmill-ai/bedrock", "windmill-api/bedrock", "windmill-worker/bedro mqtt_trigger = ["windmill-api/mqtt_trigger"] amqp_trigger = ["windmill-api/amqp_trigger"] native_trigger = ["windmill-api/native_trigger"] -sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "windmill-api/openidconnect"] +sqs_trigger = ["windmill-api/sqs_trigger", "windmill-common/aws_auth", "openidconnect"] gcp_trigger = ["windmill-api/gcp_trigger"] azure_trigger = ["windmill-api/azure_trigger"] smtp = ["windmill-api/smtp", "windmill-common/smtp", "windmill-queue/smtp"] @@ -281,6 +281,7 @@ windmill-api = { workspace = true, default-features = false } windmill-api-agent-workers = { workspace = true, optional = true } windmill-api-scripts.workspace = true windmill-api-settings.workspace = true +windmill-audit.workspace = true windmill-worker.workspace = true windmill-indexer = { workspace = true, optional = true } windmill-autoscaling = { workspace = true, optional = true } @@ -451,7 +452,7 @@ headers = "^0" hyper = { version = "^1", features = ["full"] } hyper-tls = "^0.6" hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] } -tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] } +tokio = { version = "=1.53.2", features = ["full", "tracing", "time"] } tokio-stream = { version = "0.1.17" } tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] } @@ -488,9 +489,11 @@ arc-swap = "1" ed25519-dalek = { version = "2", features = ["rand_core"] } magic-crypt = "^3" git-version = "^0" -malachite = "=0.4.18" -malachite-bigint = "=0.2.0" -rustpython-parser = "^0" +# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly, +# and to the same release. +ruff_python_parser = "=0.0.17" +ruff_python_ast = "=0.0.17" +ruff_text_size = "=0.0.17" pep440_rs = "0.7.3" php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" } cron = "^0" @@ -519,33 +522,29 @@ aws-sdk-rds = "^1" async-trait = "0.1.88" -v8 = "=137.1.0" # Exact version NOTE: Do not forget to update version and hash in flake.nix -# deno_* pin set: deno v2.4.0 base, with deno_ast force-overridden to =0.51.0. -# Rationale: deno_ast 0.51.0 is the first version pulling swc_common =14.0.4, -# the first swc_common patch that dropped `pub use serde::__private as serde;` -# (the line that capped our workspace serde pin at =1.0.220). v2.4.0's other -# pins keep deno_tls at 0.196.0 which uses permissive `rustls ^0.23.11`, -# compatible with aws-sdk-bedrockruntime's `^0.23.31` requirement. deno_tls -# 0.198+ tightened that to exact `=0.23.28`, which would have made any -# meaningful deno bump resolver-impossible against aws-sdk. -deno_fetch = "0.233.0" -deno_tls = "0.196.0" -deno_console = "0.209.0" -deno_url = "0.209.0" -deno_webidl = "0.209.0" -deno_web = "0.240.0" -# Sibling release of the pinned deno_core 0.352 / deno_web 0.240 stack -# (its deps are deno_core ^0.352, deno_web ^0.240, deno_error =0.6.1). A newer -# deno_crypto would force bumping the whole deno stack. -deno_crypto = "0.223.0" -deno_io = "0.119.0" -deno_fs = "0.119.0" -deno_net = "0.201.0" -deno_core = "0.352.0" +# deno_* pin set: the crates deno published together on 2026-09-16 (deno_core +# 0.412). +# V8 is pinned exactly because flake.nix fetches the prebuilt archive of that +# version: update its version and hash there with any bump. No default features, +# so the pin leaves the feature set deno_core picks untouched. +v8 = { version = "=150.4.0", default-features = false } +# These release as a matched set: each pins its siblings, deno_error exactly, +# and deno_tls pins rustls exactly (=0.23.40, the workspace `rustls` pin below +# must equal it). A bump moves all of them at once. +# deno_ast is independent of that set and stays at 0.51.0: a newer one pulls +# swc_common 17, which would force bumping every swc_* pin the parsers use. +deno_fetch = "0.283.0" +deno_tls = "0.246.0" +deno_webidl = "0.259.0" +deno_web = "0.290.0" +deno_crypto = "0.273.0" +deno_net = "0.251.0" +deno_core = "0.412.0" deno_ast = { version = "=0.51.0", features = ["transpiling"] } -deno_permissions = "0.68.0" -deno_telemetry = "0.31.0" -deno_error = "=0.6.1" +deno_permissions = "0.118.0" +deno_telemetry = "0.81.0" +deno_error = "=0.7.1" +sys_traits = { version = "0.1.28", features = ["real", "libc", "winapi"] } rustls-pemfile = "2.2.0" # only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed @@ -596,9 +595,11 @@ const_format = { version = "0.2.35", features = ["rust_1_64", "rust_1_51"] } const-str = "0.5" constant_time_eq = "0.3.1" rsa = "^0" +p256 = "0.13" spki = { version = "0.7", features = ["pem"] } pkcs1 = "0.7" aes-gcm = "0.10.3" +hkdf = "0.12.4" async_zip = { version = "0.0.17", features = ["tokio", "tokio-fs", "deflate", "chrono"] } once_cell = "1.17.1" dashmap = "6.1.0" @@ -682,7 +683,7 @@ aws-sdk-sqs = "^1.89.0" aws-sdk-sts = "^1.91.0" aws-sdk-sso = "=1.77.0" aws-sdk-ssooidc = "=1.78.0" -rustls = "=0.23.35" +rustls = "=0.23.40" async-once-cell = "0.5.4" size = "0.5.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 9922b6f09d..2086919735 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -44e6aa910f4a6db94532b2d08919721d86d22bc8 \ No newline at end of file +ac721c18c37bb0fd41aae17e8b4439c09168511f diff --git a/backend/migrations/20261008070335_worker_ping_draining.down.sql b/backend/migrations/20261008070335_worker_ping_draining.down.sql new file mode 100644 index 0000000000..64a5f2949d --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.down.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping DROP COLUMN IF EXISTS draining; diff --git a/backend/migrations/20261008070335_worker_ping_draining.up.sql b/backend/migrations/20261008070335_worker_ping_draining.up.sql new file mode 100644 index 0000000000..33b5ad5e3e --- /dev/null +++ b/backend/migrations/20261008070335_worker_ping_draining.up.sql @@ -0,0 +1 @@ +ALTER TABLE worker_ping ADD COLUMN IF NOT EXISTS draining BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/backend/migrations/20261008165042_runnable_job_stats.down.sql b/backend/migrations/20261008165042_runnable_job_stats.down.sql new file mode 100644 index 0000000000..21208736b4 --- /dev/null +++ b/backend/migrations/20261008165042_runnable_job_stats.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS runnable_job_stats; diff --git a/backend/migrations/20261008165042_runnable_job_stats.up.sql b/backend/migrations/20261008165042_runnable_job_stats.up.sql new file mode 100644 index 0000000000..cf676a037e --- /dev/null +++ b/backend/migrations/20261008165042_runnable_job_stats.up.sql @@ -0,0 +1,17 @@ +-- Hourly resource rollup per runnable and worker group, fed by the workers. +-- Memory figures are in kB, like v2_job_completed.memory_peak. +-- No foreign key to workspace: every worker upserts here, and the key would cost each row +-- a lookup and a lock on its workspace. Workspace deletion removes the rows itself. +CREATE TABLE IF NOT EXISTS runnable_job_stats ( + hour BIGINT NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + runnable_path VARCHAR(255) NOT NULL, + worker_group TEXT NOT NULL, + job_count INTEGER NOT NULL DEFAULT 0, + total_duration_ms BIGINT NOT NULL DEFAULT 0, + max_memory_peak INTEGER NOT NULL DEFAULT 0, + sum_memory_peak BIGINT NOT NULL DEFAULT 0, + memory_sample_count INTEGER NOT NULL DEFAULT 0, + total_cpu_ms BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (hour, workspace_id, runnable_path, worker_group) +); diff --git a/backend/migrations/20261012180101_datatable_role_set.down.sql b/backend/migrations/20261012180101_datatable_role_set.down.sql new file mode 100644 index 0000000000..15384edae8 --- /dev/null +++ b/backend/migrations/20261012180101_datatable_role_set.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS datatable_role_set; diff --git a/backend/migrations/20261012180101_datatable_role_set.up.sql b/backend/migrations/20261012180101_datatable_role_set.up.sql new file mode 100644 index 0000000000..0093837d3b --- /dev/null +++ b/backend/migrations/20261012180101_datatable_role_set.up.sql @@ -0,0 +1,21 @@ +-- The logins behind a data table connection made as several roles at once: one Postgres login per +-- distinct set of data table roles, a member of each. Which roles a login holds is its +-- memberships on the cluster, and its name is derived from their catalog ids, so a row only has to +-- carry the credential. +-- +-- A row is written after its login exists and removed before the login is dropped, so a row always +-- names a login that can authenticate with `pwd`. +CREATE TABLE datatable_role_set ( + cluster VARCHAR(20) NOT NULL CHECK (cluster IN ('instance', 'external_instance')), + name VARCHAR(63) NOT NULL, + -- Generated by Windmill, never entered by anyone, and never leaves the server. + pwd TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + -- Recorded at most once a day. A login unused for long enough is dropped, and made again if + -- its set of roles is asked for later. + last_used_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (cluster, name) +); + +GRANT ALL ON datatable_role_set TO windmill_user; +GRANT ALL ON datatable_role_set TO windmill_admin; diff --git a/backend/migrations/20261012213357_widen_variable_value.down.sql b/backend/migrations/20261012213357_widen_variable_value.down.sql new file mode 100644 index 0000000000..48b61c3f65 --- /dev/null +++ b/backend/migrations/20261012213357_widen_variable_value.down.sql @@ -0,0 +1,12 @@ +DO $$ +DECLARE + too_long BIGINT; +BEGIN + SELECT count(*) INTO too_long FROM variable WHERE length(value) > 15000; + IF too_long > 0 THEN + RAISE EXCEPTION 'cannot narrow variable.value to VARCHAR(15000): % row(s) are longer than 15000 characters', too_long; + END IF; +END $$; + +ALTER TABLE variable DROP CONSTRAINT IF EXISTS variable_value_length; +ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15000); diff --git a/backend/migrations/20261012213357_widen_variable_value.up.sql b/backend/migrations/20261012213357_widen_variable_value.up.sql new file mode 100644 index 0000000000..06dc8a191a --- /dev/null +++ b/backend/migrations/20261012213357_widen_variable_value.up.sql @@ -0,0 +1,8 @@ +-- AES-256-GCM ciphertext is 32 bytes (magic, nonce, tag) longer than legacy CBC before base64, +-- so the longest CBC value that fits 15000 chars is exactly 15040 chars once re-encrypted. +ALTER TABLE variable ALTER COLUMN value TYPE VARCHAR(15040); + +-- Only GCM secrets (base64 of the `wmA2` magic starts `d21BM`) may use the extra room: a longer +-- CBC value could never be re-encrypted to fit, and would fail every rotation of its workspace key. +ALTER TABLE variable ADD CONSTRAINT variable_value_length + CHECK (length(value) <= 15000 OR (is_secret AND value LIKE 'd21BM%')); diff --git a/backend/parsers/windmill-parser-py-asset/Cargo.toml b/backend/parsers/windmill-parser-py-asset/Cargo.toml index ddd6446079..84562dcf0e 100644 --- a/backend/parsers/windmill-parser-py-asset/Cargo.toml +++ b/backend/parsers/windmill-parser-py-asset/Cargo.toml @@ -11,6 +11,6 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true windmill-parser-sql-asset.workspace = true -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true anyhow.workspace = true diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 9c70b93d64..2919b74607 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -1,5 +1,8 @@ -use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; -use rustpython_parser::{ast::Suite, Parse}; +use ruff_python_ast::{ + visitor::{walk_expr, walk_stmt, Visitor}, + Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign, +}; +use ruff_python_parser::parse_module; use std::collections::HashMap; use windmill_parser::asset_parser::{ asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, @@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{ use AssetUsageAccessType::*; pub fn parse_assets(input: &str) -> anyhow::Result { - let ast = Suite::parse(input, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(input) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() }; - ast.into_iter() - .for_each(|stmt| assets_finder.visit_stmt(stmt)); + assets_finder.visit_body(&ast); for (kind, path, _) in assets_finder.var_identifiers.into_values() { // if a db = wmill.datatable() was never used (e.g db.query(...)), @@ -43,13 +46,72 @@ struct AssetsFinder { var_identifiers: HashMap, } -impl Visitor for AssetsFinder { +impl<'a> Visitor<'a> for AssetsFinder { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::Assign(node) = stmt { + self.track_assignment(node); + } + // Continue with generic visit to catch any other assets in the statement + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + // A detected asset call does not recurse, + // so this is only reached when no further context was found + Expr::StringLiteral(s) => self.visit_str(s.value.to_str()), + Expr::FString(f) => { + // Literal text between two interpolations reads as one string, across the + // pieces of an implicitly concatenated f-string too. + let mut literal = String::new(); + for part in &f.value { + match part { + FStringPartRef::Literal(s) => literal.push_str(&s.value), + FStringPartRef::FString(f) => { + for element in &f.elements { + match element { + InterpolatedStringElement::Literal(l) => { + literal.push_str(&l.value) + } + InterpolatedStringElement::Interpolation(_) => { + self.visit_str(&std::mem::take(&mut literal)) + } + } + } + } + } + } + self.visit_str(&literal); + walk_expr(self, expr); + } + Expr::Call(node) => { + if self.visit_expr_call_inner(node).is_err() { + walk_expr(self, expr); + } + } + _ => walk_expr(self, expr), + } + } +} + +impl AssetsFinder { + fn visit_str(&mut self, s: &str) { + if let Some((kind, path)) = parse_asset_syntax(s, false) { + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type: None, + columns: None, + }); + } + } + // Handle assignment statements like: x = wmill.datatable('name') - fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) { + fn track_assignment(&mut self, node: &StmtAssign) { // Check if the value is a call to a tracked function if let Some(Expr::Name(expr_name)) = node.targets.first() { // Remove any tracked variables with that name in case of reassignment - let Ok(var_name) = expr_name.id.parse::(); + let var_name = expr_name.id.to_string(); let removed = self.var_identifiers.remove(&var_name); // if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)), // we still want to register the asset as unknown access type @@ -69,59 +131,12 @@ impl Visitor for AssetsFinder { if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) { // Track target variable - let Ok(var_name) = expr_name.id.parse::(); self.var_identifiers .insert(var_name, (kind.clone(), name.clone(), schema.clone())); } } - // Continue with generic visit to catch any other assets in the expression - self.generic_visit_stmt_assign(node); } - // visit_call_expr will not recurse if it detects an asset, - // so this will only be called when no further context was found - fn visit_expr_constant(&mut self, node: ExprConstant) { - match node.value { - Constant::Str(s) => { - if let Some((kind, path)) = parse_asset_syntax(&s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - _ => self.generic_visit_expr_constant(node), - } - } - - fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) { - match self.visit_expr_call_inner(&node) { - Ok(_) => {} - Err(_) => { - // Check keyword arguments for assets before falling back to generic visit - for keyword in &node.keywords { - if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) = - &keyword.value - { - if let Some((kind, path)) = parse_asset_syntax(s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - } - self.generic_visit_expr_call(node); - } - } - } -} - -impl AssetsFinder { /// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc. fn extract_asset_from_call( &self, @@ -147,18 +162,20 @@ impl AssetsFinder { // Get the first argument (the asset name) let name = call + .arguments .args .first() .or_else(|| { // Try keyword argument 'name' or 'path' - call.keywords + call.arguments + .keywords .iter() .find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path"))) .map(|kw| &kw.value) }) .and_then(|first_arg| { - if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg { - Some(name.clone()) + if let Expr::StringLiteral(name) = first_arg { + Some(name.value.to_str().to_string()) } else { None } @@ -179,44 +196,39 @@ impl AssetsFinder { Some((kind, name, schema)) } - fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> { + fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> { let ident: String = node .func .as_name_expr() - .and_then(|o| o.id.parse().ok()) + .map(|o| o.id.to_string()) .or_else(|| { node.func .as_attribute_expr() - .and_then(|attr| attr.attr.parse().ok()) + .map(|attr| attr.attr.to_string()) }) .ok_or(())?; // `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()` - let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute { - value, .. - }) = node.func.as_ref() - { - if let Expr::Name(expr_name) = value.as_ref() { - expr_name.id.parse().map_err(|_| ())? - } else { - return Err(()); - } - } else { - return Err(()); - }; + let obj_name: String = node + .func + .as_attribute_expr() + .and_then(|attr| attr.value.as_name_expr()) + .map(|name| name.id.to_string()) + .ok_or(())?; if obj_name == "wmill" { // Continue } else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) { if ident == "query" { - let expr_name = node.args.get(0).or_else(|| { - node.keywords + let expr_name = node.arguments.args.get(0).or_else(|| { + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some("name")) .map(|kw| &kw.value) }); let sql = match expr_name { - Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql, + Some(Expr::StringLiteral(sql)) => sql.value.to_str(), _ => return Err(()), }; // We use the SQL parser to detect RW, specific tables, etc. @@ -224,7 +236,7 @@ impl AssetsFinder { *kind, path, schema.as_deref(), - &sql, + sql, ); match sql_assets { Ok(Some(sql_assets)) => self.assets.extend(sql_assets), @@ -252,9 +264,10 @@ impl AssetsFinder { }; let arg_val = match arg { - Arg(i, name) => node.args.get(i).or_else(|| { + Arg(i, name) => node.arguments.args.get(i).or_else(|| { // Get arg by name - node.keywords + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .map(|kw| &kw.value) @@ -270,8 +283,9 @@ impl AssetsFinder { self.assets .push(ParseAssetsResult { kind, path, access_type, columns: None }); } - Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { - let path = parse_asset_syntax(&value, false) + Some(Expr::StringLiteral(value)) => { + let value = value.value.to_str(); + let path = parse_asset_syntax(value, false) .map(|(_, p)| p) .unwrap_or(std::borrow::Cow::Borrowed(value)); self.assets.push(ParseAssetsResult { @@ -291,26 +305,24 @@ impl AssetsFinder { struct Arg(usize, &'static str); /// Extract a string-literal keyword argument, e.g. `s3="value"` in a call. -fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option { +fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option { keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .and_then(|kw| match &kw.value { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) } /// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`. -fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option { - dict.keys +fn dict_str_value(dict: &ExprDict, name: &str) -> Option { + dict.items .iter() - .zip(dict.values.iter()) - .find_map(|(key, value)| match (key.as_ref()?, value) { - ( - Expr::Constant(ExprConstant { value: Constant::Str(k), .. }), - Expr::Constant(ExprConstant { value: Constant::Str(v), .. }), - ) if k.as_str() == name => Some(v.clone()), + .find_map(|item| match (item.key.as_ref()?, &item.value) { + (Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => { + Some(v.value.to_str().to_string()) + } _ => None, }) } @@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option /// matches the TS object form and the `# on s3:///…` trigger form exactly. fn s3_object_arg_path(expr: &Expr) -> Option { let uri = match expr { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { + let s = s.value.to_str(); match s .strip_prefix("s3://") .and_then(|rest| rest.split_once('/')) { - Some((_, key)) if !key.is_empty() => s.clone(), + Some((_, key)) if !key.is_empty() => s.to_string(), _ => return None, } } @@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option { if func_name != "S3Object" { return None; } - let key = keyword_str_value(&call.keywords, "s3")?; - let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default(); + let key = keyword_str_value(&call.arguments.keywords, "s3")?; + let storage = + keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default(); format!("s3://{storage}/{key}") } Expr::Dict(dict) => { @@ -822,4 +836,62 @@ def main(): },]) ); } + + #[test] + fn test_py_asset_parser_recent_syntax() { + let input = r#" +import wmill +lazy import polars + +type Rows = list[dict] + +def main[T](name: str): + try: + with wmill.load_s3_file_reader('s3:///in.csv') as f: + f.read() + except ValueError, TypeError: + pass + greeting = t"hello {name}" + return f"{wmill.get_resource("u/user/pg")["host"]}" +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/in.csv".to_string(), + access_type: Some(R), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::Resource, + path: "u/user/pg".to_string(), + access_type: None, + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_schema_qualified_db_without_table() { + let input = r#" +import wmill +def main(): + db = wmill.datatable('dt:sch') + db.query("ATTACH 'ducklake://other' AS o") + db.query("SELECT * FROM t") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()) + .map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::>()), + Ok(vec![ + (AssetKind::DataTable, "dt".to_string()), + (AssetKind::DataTable, "dt/sch.t".to_string()), + (AssetKind::Ducklake, "other".to_string()), + ]) + ); + } } diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index 565e1422bb..2ef83f5ac9 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -23,7 +23,8 @@ tracing.workspace = true [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true phf.workspace = true itertools.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index de7f51bc79..5c8de9790f 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -13,9 +13,9 @@ use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; #[cfg(not(target_arch = "wasm32"))] -use std::str::FromStr; -#[cfg(not(target_arch = "wasm32"))] use std::collections::HashMap; +#[cfg(not(target_arch = "wasm32"))] +use std::str::FromStr; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -23,11 +23,8 @@ use regex::Regex; #[cfg(target_arch = "wasm32")] use regex_lite::Regex; -use rustpython_parser::{ - ast::{Stmt, StmtImport, StmtImportFrom, Suite}, - text_size::TextRange, - Parse, -}; +use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; +use ruff_python_parser::parse_module; #[cfg(not(target_arch = "wasm32"))] use sqlx::{Pool, Postgres}; #[cfg(not(target_arch = "wasm32"))] @@ -160,6 +157,7 @@ pub struct ImportPin { } pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result> { + let full_code = code; // Use regex to safely find the main function definition let mut code = DEF_MAIN_RE .split(code) @@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result (code, parsed.into_suite()), + // The split lands on any line reading like the definition of `main`, one inside a + // string included: what precedes it is then not a module, while the whole file is. + Err(e) => match parse_module(full_code) { + Ok(parsed) => (full_code.to_string(), parsed.into_suite()), + Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()), + }, + }; // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main // but we want to match against the original code - let find_pin = |range: TextRange, key: String| { + let find_pin = |stmt_end: usize, key: String| { let hs = code .chars() - .skip(range.end().to_usize()) + .skip(stmt_end) .take_while(|e| *e != '\n') .collect::(); @@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result = ast .into_iter() .filter_map(|x| match x { - Stmt::Import(StmtImport { names, range }) => names + Stmt::Import(StmtImport { names, range, .. }) => names .get(0) - .and_then(|al| find_pin(range, al.name.to_string())) + .and_then(|al| find_pin(range.end().to_usize(), al.name.to_string())) .or(Some( names .into_iter() @@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result>(), )), - Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => { - Some(process_import( - module.map(|x| x.to_string()), - path, - i.to_usize(), - )) - } + Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some( + process_import(module.map(|x| x.to_string()), path, level as usize), + ), Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin( - range, + range.end().to_usize(), module.clone().map(|x| x.to_string()).unwrap_or_default(), ) .or(Some(process_import(module.map(|x| x.to_string()), path, 0))), @@ -351,13 +351,16 @@ async fn parse_python_imports_inner( temp_script_refs: &Option>, ) -> error::Result> { tracing::debug!("Parsing python imports for path: {}", path); - let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); + let PythonAnnotations { py310, py311, py312, py313, py314, py315, .. } = + PythonAnnotations::parse(&code); tracing::debug!( - "Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}", + "Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}, py314: {}, py315: {}", py310, py311, py312, - py313 + py313, + py314, + py315 ); let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> { @@ -373,6 +376,8 @@ async fn parse_python_imports_inner( push_version_specifiers(py311, "==3.11.*".to_owned())?; push_version_specifiers(py312, "==3.12.*".to_owned())?; push_version_specifiers(py313, "==3.13.*".to_owned())?; + push_version_specifiers(py314, "==3.14.*".to_owned())?; + push_version_specifiers(py315, "==3.15.*".to_owned())?; for x in code.lines() { if x.starts_with("# py:") || x.starts_with("#py:") { @@ -513,8 +518,14 @@ async fn parse_python_imports_inner( let mut nested = match n { NImport::Relative(rpath) => { // First try to get content from temp_script_refs cache if available - let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { - tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash); + let code_from_cache = if let Some(hash) = + temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) + { + tracing::debug!( + "Found relative import '{}' in temp_script_refs with hash '{}'", + rpath, + hash + ); match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await { Ok(content) => Some(content), Err(e) => { @@ -529,19 +540,17 @@ async fn parse_python_imports_inner( // Use cached content if available, otherwise fall back to deployed script let code = match code_from_cache { Some(content) => content, - None => { - sqlx::query_scalar!( - r#" + None => sqlx::query_scalar!( + r#" SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1 "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()) - } + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()), }; if already_visited.contains(&rpath) { diff --git a/backend/parsers/windmill-parser-py-imports/src/mapping.rs b/backend/parsers/windmill-parser-py-imports/src/mapping.rs index 6246a1f156..90f3fab99c 100644 --- a/backend/parsers/windmill-parser-py-imports/src/mapping.rs +++ b/backend/parsers/windmill-parser-py-imports/src/mapping.rs @@ -383,5 +383,10 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! { "vt" => "vt-py", "grpc" => "grpcio", "serpapi" => "google-search-results", + "pptx" => "python-pptx", + "OpenSSL" => "pyOpenSSL", + "win32api" => "pywin32", + "ruamel" => "ruamel.yaml", + "speech-recognition" => "SpeechRecognition", // Add new entry here ^ }; diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index c853c8e5b4..9086b29d47 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -1,7 +1,9 @@ mod tests { use sqlx::{Pool, Postgres}; - use windmill_parser_py_imports::parse_python_imports; + use windmill_parser_py_imports::{ + parse_code_for_imports, parse_python_imports, ImportPin, NImport, + }; #[sqlx::test(fixtures("base"))] async fn test_parse_python_imports(db: Pool) -> anyhow::Result<()> { @@ -117,4 +119,87 @@ def main(): Ok(()) } + + fn auto(pkg: &str) -> NImport { + NImport::Auto { pkg: pkg.to_string(), key: None } + } + + #[test] + fn test_parse_lazy_imports() -> anyhow::Result<()> { + let code = " +lazy import requests +lazy import numpy # pin: numpy==2.1.0 +lazy from pandas import DataFrame +lazy from f.shared import helper +lazy = 1 + +def main(): + pass +"; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![ + NImport::Pin { + pins: vec![ImportPin { + pkg: "numpy==2.1.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "numpy".to_string() + }, + auto("pandas"), + auto("requests"), + NImport::Relative("f/shared".to_string()), + ] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> { + let code = r#" +import requests + +type Alias = list[int] +NAME = "x" +D = {"a": 1} +L = [[1], [2]] +TEMPLATE = t"hello {NAME}" +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +try: + import ujson +except ImportError, ValueError: + pass + +def main(): + pass +"#; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![auto("requests")] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n" + ); + assert_eq!( + parse_code_for_imports(&code, "f/foo/bar")?, + vec![NImport::Pin { + pins: vec![ImportPin { + pkg: "requests==2.32.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "requests".to_string() + }], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 0c50e0f8d8..6971fe8fcd 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -10,8 +10,8 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true itertools.workspace = true serde_json.workspace = true anyhow.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 086a354d02..5a9eabc33e 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -13,13 +13,11 @@ use itertools::Itertools; use serde_json::json; use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ}; -use rustpython_parser::{ - ast::{ - Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt, - StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite, - }, - Parse, +use ruff_python_ast::{ + Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign, + StmtClassDef, StmtFunctionDef, }; +use ruff_python_parser::parse_module; pub mod pydantic_parser; @@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool { || code.contains("import dataclasses") } +/// What follows `def ` or `async def ` on a line opening that function's +/// parameters or type parameters. +fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> { + let rest = line + .strip_prefix("async def ") + .or_else(|| line.strip_prefix("def "))? + .strip_prefix(name)?; + (rest.starts_with('(') || rest.starts_with('[')).then_some(rest) +} + fn filter_non_main(code: &str, main_name: &str) -> String { - let def_main = format!("def {}(", main_name); - let async_def_main = format!("async def {}(", main_name); let mut filtered_code = String::new(); let mut code_iter = code.split("\n"); let mut remaining: String = String::new(); while let Some(line) = code_iter.next() { - if line.starts_with(&async_def_main) { - filtered_code += &async_def_main; - remaining += line.strip_prefix(&async_def_main).unwrap(); - remaining += "\n"; - remaining += &code_iter.join("\n"); - break; - } else if line.starts_with(&def_main) { - filtered_code += &def_main; - remaining += line.strip_prefix(&def_main).unwrap(); + if let Some(rest) = strip_def(line, main_name) { + filtered_code += &line[..line.len() - rest.len()]; + remaining += rest; remaining += "\n"; remaining += &code_iter.join("\n"); break; @@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String { return String::new(); } let mut chars = remaining.chars(); + + // PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters. + if remaining.starts_with('[') { + let mut open_brackets = 0; + for c in chars.by_ref() { + if c == '[' { + open_brackets += 1; + } else if c == ']' { + open_brackets -= 1; + } + filtered_code.push(c); + if open_brackets == 0 { + break; + } + } + } + if chars.next() != Some('(') { + return String::new(); + } + filtered_code.push('('); let mut open_parens = 1; while let Some(c) = chars.next() { @@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let filtered_code = filter_relevant_statements(code); - let ast = match Suite::parse(&filtered_code, "main.py") { - Ok(ast) => ast, + let ast = match parse_module(&filtered_code) { + Ok(parsed) => parsed.into_suite(), Err(_) => return CodeMetadata { enums, descriptions }, }; for stmt in ast { match stmt { - Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => { - let is_enum = bases.iter().any(|base| { + Stmt::ClassDef(class_def) if has_enum => { + let is_enum = class_def.bases().iter().any(|base| { matches!(base, Expr::Name(ExprName { id, .. }) if id == "Enum" || id == "IntEnum" || id == "StrEnum" || id == "Flag" || id == "IntFlag") @@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let mut values = Vec::new(); let mut members = HashMap::new(); + let StmtClassDef { name, body, .. } = class_def; for item in body { if let Stmt::Assign(StmtAssign { targets, value, .. }) = item { if let Some(Expr::Name(ExprName { id: target_name, .. })) = targets.first() { if !target_name.starts_with('_') { - if let Expr::Constant(ExprConstant { - value: Constant::Str(val), - .. - }) = value.as_ref() - { + if let Expr::StringLiteral(val) = value.as_ref() { + let val = val.value.to_str(); values.push(val.to_string()); members.insert(target_name.to_string(), val.to_string()); } @@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { } } } - Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => { - if &func_name == main_name { + Stmt::FunctionDef(StmtFunctionDef { + name: func_name, body, is_async: false, .. + }) if has_docstring => { + if func_name.as_str() == main_name { if let Some(Stmt::Expr(expr_stmt)) = body.first() { - if let Expr::Constant(ExprConstant { - value: Constant::Str(docstring), - .. - }) = expr_stmt.value.as_ref() - { - descriptions = parse_docstring_args(docstring); + if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() { + descriptions = parse_docstring_args(docstring.value.to_str()); } } } @@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap { descriptions } +/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring +/// showing an example for instance. When several lines do, only the parsed module tells which +/// one is the function. +fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option { + code.split('\n') + .filter(|line| strip_def(line, main_name).is_some()) + .nth(1)?; + parse_module(code) + .ok()? + .into_suite() + .into_iter() + .find_map(|x| match x { + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => + { + Some(*parameters) + } + _ => None, + }) +} + /// skip_params is a micro optimization for when we just want to find the main /// function without parsing all the params. pub fn parse_python_signature( @@ -310,41 +347,39 @@ pub fn parse_python_signature( // - If no models: parse only the filtered main function let (params, module) = if should_parse_for_models(code) { // Parse full code once for both Pydantic detection and signature extraction - let ast = Suite::parse(code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); // Extract main function from full AST let params = ast.iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => { - Some(args.as_ref().clone()) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(args.as_ref().clone()) + Some(parameters.as_ref().clone()) } _ => None, }); // Keep AST for Pydantic/dataclass detection (params, Some(ast)) + } else if let Some(params) = main_params_among_lookalikes(code, &main_name) { + (Some(params), None) } else { // No models detected - parse only the filtered main function let filtered_code = filter_non_main(code, &main_name); if filtered_code.is_empty() { (None, None) } else { - let ast = Suite::parse(&filtered_code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(&filtered_code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let params = ast.into_iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => { - Some(*args) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if &name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(*args) + Some(*parameters) } _ => None, }); @@ -378,8 +413,6 @@ pub fn parse_python_signature( if !skip_params && params.is_some() { let params = params.unwrap(); - let def_arg_start = params.args.len() - params.defaults().count(); - // Two-pass approach for lazy metadata extraction: // Pass 1: Parse types without enum info to determine if metadata is needed // Pass 2: Re-parse unknown types with metadata only if necessary @@ -392,11 +425,11 @@ pub fn parse_python_signature( .iter() .enumerate() .map(|(i, x)| { - let arg_name = x.as_arg().arg.to_string(); + let arg_name = x.parameter.name.to_string(); let (typ, has_default) = x - .as_arg() + .parameter .annotation - .as_ref() + .as_deref() .map_or((Typ::Unknown, false), |e| { parse_expr(e, &empty_enums, module_ref) }); @@ -423,21 +456,16 @@ pub fn parse_python_signature( .into_iter() .map(|(i, arg_name, mut typ, mut has_default)| { if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() { - if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() { + if let Some(annotation) = params.args[i].parameter.annotation.as_deref() { (typ, has_default) = parse_expr(annotation, &metadata.enums, module_ref); } } - let default = if i >= def_arg_start { - params - .defaults() - .nth(i - def_arg_start) - .map(|expr| to_value(expr, &metadata.enums)) - .flatten() - } else { - None - }; + let default = params.args[i] + .default + .as_deref() + .and_then(|expr| to_value(expr, &metadata.enums)); let should_get_type_from_default = match &typ { Typ::Unknown => true, @@ -465,10 +493,10 @@ pub fn parse_python_signature( // debounce accumulation support. Falls back to docstring // description if no union array pattern is found. let union_otyp = params.args[i] - .as_arg() + .parameter .annotation - .as_ref() - .and_then(|ann| detect_py_union_array_otyp(ann.as_ref())); + .as_deref() + .and_then(detect_py_union_array_otyp); Arg { otyp: union_otyp.or_else(|| { @@ -509,12 +537,8 @@ pub fn parse_python_signature( } } -fn parse_expr( - e: &Box, - enums: &HashMap, - module: Option<&[Stmt]>, -) -> (Typ, bool) { - match e.as_ref() { +fn parse_expr(e: &Expr, enums: &HashMap, module: Option<&[Stmt]>) -> (Typ, bool) { + match e { Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false), Expr::Attribute(x) => { if let Some(name) = x.value.as_name_expr() { @@ -531,12 +555,9 @@ fn parse_expr( } } Expr::BinOp(x) => { - if matches!( - x.right.as_ref(), - Expr::Constant(ExprConstant { value: Constant::None, .. }) - ) { + if x.right.is_none_literal_expr() { (parse_expr(&x.left, enums, module).0, true) - } else if detect_py_union_array_otyp(e.as_ref()).is_some() { + } else if detect_py_union_array_otyp(e).is_some() { // T | list[T] — parsed type is Unknown; otyp is set separately (Typ::Unknown, false) } else { @@ -552,7 +573,7 @@ fn parse_expr( .elts .iter() .map(|x| match x { - Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) .filter_map(|x| x) @@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String { } } -fn to_value(et: &Expr, enums: &HashMap) -> Option { +fn to_value(et: &Expr, enums: &HashMap) -> Option { match et { - Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)), - Expr::Dict(ExprDict { keys, values, .. }) => { - let v = keys - .into_iter() - .zip(values) - .map(|(k, v)| { - let key = k + Expr::NoneLiteral(_) => Some(json!(null)), + Expr::BooleanLiteral(b) => Some(json!(b.value)), + Expr::StringLiteral(s) => Some(json!(s.value.to_str())), + Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::>())), + Expr::NumberLiteral(n) => Some(match &n.value { + Number::Int(i) => { + serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")) + } + Number::Float(f) => json!(f), + Number::Complex { real, imag } => json!([real, imag]), + }), + Expr::EllipsisLiteral(_) => Some(json!("...")), + Expr::Dict(ExprDict { items, .. }) => { + let v = items + .iter() + .map(|item| { + let key = item + .key .as_ref() .map(|e| to_value(e, enums)) .flatten() @@ -642,7 +674,7 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option None, }) .unwrap_or_else(|| "no_key".to_string()); - (key, to_value(&v, enums)) + (key, to_value(&item.value, enums)) }) .collect::>(); Some(json!(v)) @@ -672,20 +704,6 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option serde_json::Value { - match c { - Constant::None => json!(null), - Constant::Bool(b) => json!(b), - Constant::Str(s) => json!(s), - Constant::Bytes(b) => json!(b), - Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")), - Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::>()), - Constant::Float(f) => json!(f), - Constant::Complex { real, imag } => json!([real, imag]), - Constant::Ellipsis => json!("..."), - } -} - #[cfg(test)] mod tests { @@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED): ); Ok(()) } + + #[test] + fn test_parse_python_sig_type_params() -> anyhow::Result<()> { + for code in [ + "def main[T](x: str, y: int = 3):\n return x\n", + "async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n", + // parsed whole rather than pre-filtered down to `main` + "from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n", + ] { + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.auto_kind, None, "{code}"); + assert_eq!( + result + .args + .iter() + .map(|a| (a.name.as_str(), &a.typ, &a.default)) + .collect::>(), + vec![ + ("x", &Typ::Str(None), &None), + ("y", &Typ::Int, &Some(json!(3))) + ], + "{code}" + ); + } + Ok(()) + } + + // A script with a model class is parsed whole, so syntax anywhere in it reaches the parser. + #[test] + fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> { + let code = r#" +from dataclasses import dataclass +lazy import requests +lazy from pandas import DataFrame + +type Alias = list[int] + +@dataclass +class Point: + x: int + +try: + import a +except ImportError, ValueError: + pass + +D = {"a": 1} +L = [[1], [2]] +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +def main(p: Point, name: str = "world"): + return t"hello {name}" +"#; + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.args.len(), 2); + assert_eq!( + result.args[0].typ, + Typ::Object(ObjectType::new( + Some("Point".to_string()), + Some(vec![windmill_parser::ObjectProperty { + key: "x".to_string(), + typ: Box::new(Typ::Int) + }]) + )) + ); + assert_eq!(result.args[1].default, Some(json!("world"))); + Ok(()) + } + + #[test] + fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x: int, y: int):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n" + ); + let result = parse_python_signature(&code, None, false)?; + assert_eq!( + result.args.iter().map(|a| &a.name).collect::>(), + vec!["value"], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs index e229d97a8a..78a0a3c354 100644 --- a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs +++ b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs @@ -11,8 +11,8 @@ //! This module provides functionality to detect and parse Pydantic models and Python //! dataclasses from Python AST, enabling automatic UI generation for complex data structures. -use rustpython_parser::ast::{ - Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign, +use ruff_python_ast::{ + Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign, }; use std::collections::HashSet; use windmill_parser::{ObjectProperty, ObjectType, Typ}; @@ -64,7 +64,7 @@ fn detect_model_type_impl( // Mark as being visited visited.insert(class_name.to_string()); - let result = if is_pydantic_base(&class_def.bases) { + let result = if is_pydantic_base(class_def.bases()) { // Pydantic BaseModel parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited) } else if has_dataclass_decorator(&class_def.decorator_list) @@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool { } /// Checks if a class has @dataclass decorator (standard library) -fn has_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @dataclass Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => { return true; @@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool { } /// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2) -fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @pydantic.dataclasses.dataclass Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => { if let Expr::Attribute(ExprAttribute { @@ -350,10 +350,10 @@ fn extract_field_type( // String annotations: "ForwardRef" (forward references) // Not fully supported - return Unknown with warning - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { eprintln!( "Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown", - forward_ref = s, + forward_ref = s.value.to_str(), ); Typ::Unknown } diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs index b29c6c99ac..4d77da15bb 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs @@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets( Ok(mut sql_assets) => { if let Some(schema) = schema { for asset in &mut sql_assets.assets { - if asset.kind == kind && asset.path.starts_with(asset_name) { - asset.path = format!( - "{}/{}.{}", - asset_name, - schema, - &asset.path[asset_name.len() + 1..] - ); + if asset.kind != kind { + continue; + } + // A statement can name the database alone, with no table to qualify. + let table = asset + .path + .strip_prefix(asset_name) + .and_then(|rest| rest.strip_prefix('/')); + if let Some(table) = table { + asset.path = format!("{asset_name}/{schema}.{table}"); } } } diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index 10929e409a..341906a3e3 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -235,7 +235,7 @@ lazy_static::lazy_static! { // -- $name (type) = default static ref RE_ARG_DUCKDB: Regex = Regex::new(r#"(?m)^-- \$(\w+) \(([A-Za-z0-9_\[\]]+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); - static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); + static ref RE_ARG_SNOWFLAKE: Regex = Regex::new(r#"(?m)^-- \? (\w+) \((\w+(?:\[\])?)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); static ref RE_ARG_MSSQL: Regex = Regex::new(r#"(?m)^-- @(?:P|p)\d+ (\w+) \((\w+)\)(?: ?\= ?(.+))? *(?:\r|\n|$)"#).unwrap(); @@ -254,6 +254,12 @@ fn parsed_default(parsed_typ: &Typ, default: String) -> Option= 2 && default.starts_with("'") && default.ends_with("'") => { Some(json!(&default[1..default.len() - 1])) } + Typ::List(_) => Some( + serde_json::from_str::(&default) + .ok() + .filter(|v| v.is_array()) + .unwrap_or_else(|| json!(default)), + ), _ => Some(json!(default)), } } @@ -1039,7 +1045,11 @@ pub fn parse_duckdb_typ(typ: &str) -> Typ { } pub fn parse_snowflake_typ(typ: &str) -> Typ { + if let Some(base_typ) = typ.strip_suffix("[]") { + return Typ::List(Box::new(parse_snowflake_typ(base_typ))); + } match typ { + "array" => Typ::List(Box::new(Typ::Unknown)), "varchar" | "text" | "string" | "char" | "character" => Typ::Str(None), "binary" | "varbinary" => Typ::Bytes, "date" | "time" | "datetime" | "timestamp" | "timestamp_ntz" | "timestamp_ltz" @@ -1591,6 +1601,39 @@ SELECT ?; Ok(()) } + #[test] + fn test_parse_snowflake_array_types() -> anyhow::Result<()> { + let code = r#" +-- ? groups (varchar[]) +-- ? ids (int[]) = [1, 2] +-- ? anything (ARRAY) +SELECT 1; +"#; + let args = parse_snowflake_sig(code)?.args; + assert_eq!(args[1].default, Some(json!([1, 2]))); + let got = args + .iter() + .map(|a| (a.name.as_str(), a.otyp.as_deref(), &a.typ)) + .collect::>(); + assert_eq!( + got, + vec![ + ( + "groups", + Some("varchar[]"), + &Typ::List(Box::new(Typ::Str(None))) + ), + ("ids", Some("int[]"), &Typ::List(Box::new(Typ::Int))), + ( + "anything", + Some("array"), + &Typ::List(Box::new(Typ::Unknown)) + ), + ] + ); + Ok(()) + } + #[test] fn test_parse_mssql_sig() -> anyhow::Result<()> { let code = r#" diff --git a/backend/parsers/windmill-parser-wac/Cargo.toml b/backend/parsers/windmill-parser-wac/Cargo.toml index d354b1c653..21ee286db9 100644 --- a/backend/parsers/windmill-parser-wac/Cargo.toml +++ b/backend/parsers/windmill-parser-wac/Cargo.toml @@ -9,8 +9,9 @@ name = "windmill_parser_wac" path = "./src/lib.rs" [dependencies] -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true +ruff_text_size.workspace = true swc_common.workspace = true swc_ecma_parser.workspace = true swc_ecma_ast.workspace = true diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs index f91067b711..e8fb72ff2f 100644 --- a/backend/parsers/windmill-parser-wac/src/python.rs +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; -use rustpython_parser::{ - ast::{ - Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry, - StmtTryStar, StmtWhile, - }, - Parse, +use ruff_python_ast::{ + CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName, + Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry, + StmtWhile, }; +use ruff_python_parser::parse_module; +use ruff_text_size::{Ranged, TextSize}; use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; use crate::validation::{self, CompileError}; @@ -42,9 +42,9 @@ type TaskFunctions = HashMap>; fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { let mut tasks = HashMap::new(); for stmt in stmts { - if let Stmt::AsyncFunctionDef(func) = stmt { + if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt { for dec in &func.decorator_list { - match dec { + match &dec.expression { // @task (bare decorator) Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => { tasks.insert(func.name.to_string(), None); @@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { if id.as_str() == "task_script" || id.as_str() == "task_flow" { // Extract the path from the first positional argument - let path = call.args.first().and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + let path = call.arguments.args.first().and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }); // Extract variable name from target if let Some(Expr::Name(ExprName { id: var_name, .. })) = @@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { /// Extract the `path=` keyword argument from a `@task(path="...")` call. fn extract_task_path_kwarg(call: &ExprCall) -> Option { - for kw in &call.keywords { + for kw in &call.arguments.keywords { if let Some(ref arg) = kw.arg { if arg.as_str() == "path" { - if let Expr::Constant(c) = &kw.value { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } + if let Expr::StringLiteral(s) = &kw.value { + return Some(s.value.to_str().to_string()); } } } @@ -150,10 +144,9 @@ impl WacWalker { fn line_of_expr(&self, expr: &Expr) -> usize { let offset = match expr { - Expr::Call(c) => c.range.start().to_usize(), - Expr::Await(a) => a.range.start().to_usize(), - Expr::Attribute(a) => a.range.start().to_usize(), - Expr::Name(n) => n.range.start().to_usize(), + Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => { + expr.start().to_usize() + } _ => 0, }; self.line_index.line_of(offset) @@ -161,17 +154,15 @@ impl WacWalker { fn line_of_stmt(&self, stmt: &Stmt) -> usize { let offset = match stmt { - Stmt::If(s) => s.range.start().to_usize(), - Stmt::For(s) => s.range.start().to_usize(), - Stmt::While(s) => s.range.start().to_usize(), - Stmt::Return(s) => s.range.start().to_usize(), - Stmt::Expr(s) => s.range.start().to_usize(), - Stmt::Try(s) => s.range.start().to_usize(), - Stmt::TryStar(s) => s.range.start().to_usize(), - Stmt::Assign(s) => s.range.start().to_usize(), - Stmt::AnnAssign(s) => s.range.start().to_usize(), - Stmt::FunctionDef(s) => s.range.start().to_usize(), - Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(), + Stmt::If(_) + | Stmt::For(_) + | Stmt::While(_) + | Stmt::Return(_) + | Stmt::Expr(_) + | Stmt::Try(_) + | Stmt::Assign(_) + | Stmt::AnnAssign(_) + | Stmt::FunctionDef(_) => stmt.start().to_usize(), _ => 0, }; self.line_index.line_of(offset) @@ -190,9 +181,7 @@ impl WacWalker { /// Check if an expression is `asyncio.gather(...)` call fn is_asyncio_gather_call(expr: &Expr) -> bool { if let Expr::Call(call) = expr { - if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) = - call.func.as_ref() - { + if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() { if attr.as_str() == "gather" { if let Expr::Name(ExprName { id, .. }) = value.as_ref() { return id.as_str() == "asyncio"; @@ -222,20 +211,20 @@ impl WacWalker { fn expr_to_source(expr: &Expr) -> String { match expr { Expr::Compare(c) => { - let left = Self::expr_to_source(&c.left); - if let Some(comparator) = c.comparators.first() { + let left = Self::expr_to_source(&c.operands[0]); + if let Some(comparator) = c.comparators().first() { let right = Self::expr_to_source(comparator); let op = match c.ops.first() { - Some(rustpython_parser::ast::CmpOp::Gt) => ">", - Some(rustpython_parser::ast::CmpOp::Lt) => "<", - Some(rustpython_parser::ast::CmpOp::GtE) => ">=", - Some(rustpython_parser::ast::CmpOp::LtE) => "<=", - Some(rustpython_parser::ast::CmpOp::Eq) => "==", - Some(rustpython_parser::ast::CmpOp::NotEq) => "!=", - Some(rustpython_parser::ast::CmpOp::In) => "in", - Some(rustpython_parser::ast::CmpOp::NotIn) => "not in", - Some(rustpython_parser::ast::CmpOp::Is) => "is", - Some(rustpython_parser::ast::CmpOp::IsNot) => "is not", + Some(CmpOp::Gt) => ">", + Some(CmpOp::Lt) => "<", + Some(CmpOp::GtE) => ">=", + Some(CmpOp::LtE) => "<=", + Some(CmpOp::Eq) => "==", + Some(CmpOp::NotEq) => "!=", + Some(CmpOp::In) => "in", + Some(CmpOp::NotIn) => "not in", + Some(CmpOp::Is) => "is", + Some(CmpOp::IsNot) => "is not", None => "?", }; format!("{left} {op} {right}") @@ -253,14 +242,14 @@ impl WacWalker { format!("{value}.{}", a.attr) } Expr::Name(n) => n.id.to_string(), - Expr::Constant(c) => match &c.value { - rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""), - rustpython_parser::ast::Constant::Int(i) => i.to_string(), - rustpython_parser::ast::Constant::Float(f) => f.to_string(), - rustpython_parser::ast::Constant::Bool(b) => b.to_string(), - rustpython_parser::ast::Constant::None => "None".to_string(), - _ => "...".to_string(), + Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()), + Expr::NumberLiteral(n) => match &n.value { + Number::Int(i) => i.to_string(), + Number::Float(f) => f.to_string(), + Number::Complex { .. } => "...".to_string(), }, + Expr::BooleanLiteral(b) => b.value.to_string(), + Expr::NoneLiteral(_) => "None".to_string(), _ => "...".to_string(), } } @@ -279,8 +268,16 @@ impl WacWalker { match stmt { Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value), Stmt::Assign(a) => self.expr_contains_step(&a.value), - Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), - Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::If(s) => { + self.body_contains_step(&s.body) + || s.elif_else_clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)) + } + // `async for` is not modelled: its body is skipped like any unsupported statement + Stmt::For(s) if !s.is_async => { + self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) + } Stmt::While(s) => { self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) } @@ -289,19 +286,7 @@ impl WacWalker { || self.body_contains_step(&s.orelse) || self.body_contains_step(&s.finalbody) || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }) - } - Stmt::TryStar(s) => { - self.body_contains_step(&s.body) - || self.body_contains_step(&s.orelse) - || self.body_contains_step(&s.finalbody) - || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }) } Stmt::Return(_) => false, @@ -319,11 +304,12 @@ impl WacWalker { match expr { Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), Expr::Call(call) => { - if self.is_task_fn_call(&Expr::Call(call.clone())) { - return true; - } - if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) { - return call.args.iter().any(|a| self.expr_contains_step(a)); + if Self::is_asyncio_gather_call(expr) { + return call + .arguments + .args + .iter() + .any(|a| self.expr_contains_step(a)); } false } @@ -370,12 +356,11 @@ impl WacWalker { Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value), Stmt::Assign(a) => self.walk_expr_stmt(&a.value), Stmt::If(if_stmt) => self.walk_if(if_stmt), - Stmt::For(for_stmt) => self.walk_for(for_stmt), + Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt), Stmt::While(while_stmt) => self.walk_while(while_stmt), Stmt::Try(try_stmt) => self.walk_try(try_stmt), - Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt), Stmt::Return(ret) => self.walk_return(ret), - Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => { + Stmt::FunctionDef(_) => { if self.stmt_contains_step(stmt) { self.errors.push(validation::error_step_in_nested_function( self.line_of_stmt(stmt), @@ -392,7 +377,7 @@ impl WacWalker { if let Expr::Await(ExprAwait { value, .. }) = expr { // await task_fn(...) if let Expr::Call(call) = value.as_ref() { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(value) { return self.emit_step(call, expr); } // Check for SDK-level calls: step(), sleep(), wait_for_approval() @@ -430,15 +415,12 @@ impl WacWalker { "step" => { // step("name", fn) — extract the name from the first string argument let name = call + .arguments .args .first() - .and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + .and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }) .unwrap_or_else(|| "step".to_string()); let id = self.next_id(); @@ -452,6 +434,7 @@ impl WacWalker { } "sleep" => { let seconds = call + .arguments .args .first() .map(|arg| Self::expr_to_source(arg)) @@ -517,10 +500,10 @@ impl WacWalker { }); let mut step_ids = Vec::new(); - for arg in &gather_call.args { + for arg in &gather_call.arguments.args { // Each arg should be task_fn(...) if let Expr::Call(call) = arg { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(arg) { let (name, script) = self .extract_step_info_from_task_call(call) .unwrap_or(("unknown".into(), "unknown".into())); @@ -553,15 +536,33 @@ impl WacWalker { } fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> { - let has_steps_in_body = self.body_contains_step(&if_stmt.body); - let has_steps_in_else = self.body_contains_step(&if_stmt.orelse); + self.walk_if_chain( + if_stmt.start(), + &if_stmt.test, + &if_stmt.body, + &if_stmt.elif_else_clauses, + ) + } + + /// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it. + fn walk_if_chain( + &mut self, + start: TextSize, + test: &Expr, + body: &[Stmt], + clauses: &[ElifElseClause], + ) -> Option<(String, String)> { + let has_steps_in_body = self.body_contains_step(body); + let has_steps_in_else = clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)); if !has_steps_in_body && !has_steps_in_else { return None; } - let line = self.line_index.line_of(if_stmt.range.start().to_usize()); - let condition_source = Self::expr_to_source(&if_stmt.test); + let line = self.line_index.line_of(start.to_usize()); + let condition_source = Self::expr_to_source(test); let branch_id = self.next_id(); let branch_node_id = self.add_node(DagNode { @@ -573,15 +574,19 @@ impl WacWalker { let mut last_ids = Vec::new(); - if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { + if let Some((true_first, true_last)) = self.walk_body(body) { self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); last_ids.push(true_last); } else { last_ids.push(branch_node_id.clone()); } - if !if_stmt.orelse.is_empty() { - if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) { + if let Some((clause, clauses)) = clauses.split_first() { + let orelse = match &clause.test { + Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses), + None => self.walk_body(&clause.body), + }; + if let Some((else_first, else_last)) = orelse { self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); last_ids.push(else_last); } else { @@ -611,7 +616,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(for_stmt.range.start().to_usize()); + let line = self.line_index.line_of(for_stmt.start().to_usize()); let iter_source = Self::expr_to_source(&for_stmt.iter); let start_id = self.next_id(); @@ -644,7 +649,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + let line = self.line_index.line_of(while_stmt.start().to_usize()); let condition = Self::expr_to_source(&while_stmt.test); let start_id = self.next_id(); @@ -677,39 +682,14 @@ impl WacWalker { || self.body_contains_step(&try_stmt.orelse) || self.body_contains_step(&try_stmt.finalbody) || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }); if !has_steps { return None; } - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); - self.emit_try_catch_branch( - &try_stmt.body, - &try_stmt.handlers, - &try_stmt.finalbody, - line, - ) - } - - fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { - let has_steps = self.body_contains_step(&try_stmt.body) - || self.body_contains_step(&try_stmt.orelse) - || self.body_contains_step(&try_stmt.finalbody) - || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }); - - if !has_steps { - return None; - } - - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + let line = self.line_index.line_of(try_stmt.start().to_usize()); self.emit_try_catch_branch( &try_stmt.body, &try_stmt.handlers, @@ -721,7 +701,7 @@ impl WacWalker { fn emit_try_catch_branch( &mut self, try_body: &[Stmt], - handlers: &[rustpython_parser::ast::ExceptHandler], + handlers: &[ExceptHandler], finally_body: &[Stmt], line: usize, ) -> Option<(String, String)> { @@ -746,7 +726,7 @@ impl WacWalker { // Except handlers for handler in handlers { match handler { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + ExceptHandler::ExceptHandler(eh) => { if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) { self.add_edge(&branch_node_id, &catch_first, Some("except".to_string())); last_ids.push(catch_last); @@ -783,7 +763,7 @@ impl WacWalker { } fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { - let line = self.line_index.line_of(ret.range.start().to_usize()); + let line = self.line_index.line_of(ret.start().to_usize()); let id = self.next_id(); let node_id = self.add_node(DagNode { id: id.clone(), @@ -796,12 +776,12 @@ impl WacWalker { } /// Extract workflow function parameters (no longer skips ctx) -fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { +fn extract_params(args: &Parameters) -> Vec { let mut params = Vec::new(); for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) { - let name = arg_with_default.def.arg.to_string(); + let name = arg_with_default.parameter.name.to_string(); let typ = arg_with_default - .def + .parameter .annotation .as_ref() .map(|ann| WacWalker::expr_to_source(ann)); @@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { } pub fn parse_python_workflow(code: &str) -> Result> { - let ast = rustpython_parser::ast::Suite::parse(code, "") - .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?; + let ast = parse_module(code) + .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])? + .into_suite(); // First pass: collect @task functions let task_functions = collect_task_functions(&ast); - // Find the @workflow async def - let workflow_fn = ast.iter().find_map(|stmt| { - if let Stmt::AsyncFunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return Some(func); + let workflow_fns = || { + ast.iter().filter_map(|stmt| match stmt { + Stmt::FunctionDef(func) + if func.decorator_list.iter().any(|dec| { + matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow") + }) => + { + Some(func) } - } - // Also check non-async for error reporting - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return None; // Will be reported as not-async below - } - } - None - }); + _ => None, + }) + }; - // Check for non-async workflow function - let non_async_workflow = ast.iter().find_map(|stmt| { - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - let line_index = LineIndex::new(code); - return Some(line_index.line_of(func.range.start().to_usize())); - } - } - None - }); + // Find the @workflow async def + let workflow_fn = workflow_fns().find(|func| func.is_async); + + // Check for non-async workflow function. The statement starts at its first decorator, + // the name sits on the `def` line the error points at. + let non_async_workflow = workflow_fns() + .find(|func| !func.is_async) + .map(|func| LineIndex::new(code).line_of(func.name.start().to_usize())); if let Some(line) = non_async_workflow { if workflow_fn.is_none() { @@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result String { hasher.update(code.as_bytes()); format!("{:x}", hasher.finalize()) } - -trait ToUsize { - fn to_usize(self) -> usize; -} - -impl ToUsize for rustpython_parser::text_size::TextSize { - fn to_usize(self) -> usize { - u32::from(self) as usize - } -} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs index cf117da9e7..c0de489153 100644 --- a/backend/parsers/windmill-parser-wac/tests/python_tests.rs +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -363,3 +363,77 @@ async def main(x: str): )); assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return)); } + +#[test] +fn test_elif_chain_nests_branches() { + let code = r#" +from wmill import workflow, task + +@task +async def a(): ... +@task +async def b(): ... +@task +async def c(): ... + +@workflow +async def main(x: int): + if x > 2: + await a() + elif x == 2: + await b() + else: + await c() +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + let branches: Vec<(&str, usize)> = dag + .nodes + .iter() + .filter_map(|n| match &n.node_type { + DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)), + _ => None, + }) + .collect(); + assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]); + + // The `elif` branch hangs off the `false` edge of the `if` before it + let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id); + assert!(dag + .edges + .iter() + .any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false"))); + assert_eq!( + dag.nodes + .iter() + .filter(|n| matches!(n.node_type, DagNodeType::Merge)) + .count(), + 2 + ); +} + +#[test] +fn test_workflow_with_recent_syntax() { + let code = r#" +from wmill import workflow, task +lazy import polars + +type Rows = list[dict] + +@task +async def load[T](rows: T): ... + +@workflow +async def main(name: str): + try: + await load(t"hello {name}") + except ValueError, TypeError: + pass +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert!(dag + .nodes + .iter() + .any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load"))); +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index d7e33313c6..51878c6dfb 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -2,21 +2,22 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + [[package]] name = "aes" version = "0.7.5" @@ -24,11 +25,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", - "cipher", + "cipher 0.3.0", "cpufeatures 0.2.17", "opaque-debug", ] +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher 0.4.4", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes 0.8.4", + "cipher 0.4.4", + "ctr", + "ghash", + "subtle", +] + [[package]] name = "ahash" version = "0.7.8" @@ -254,6 +280,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow", + "proc-macro2", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.7", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -360,21 +415,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - [[package]] name = "base64" version = "0.21.7" @@ -428,7 +468,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash", "shlex", "syn 2.0.117", ] @@ -510,7 +550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" dependencies = [ "block-padding", - "cipher", + "cipher 0.3.0", ] [[package]] @@ -620,6 +660,15 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.56" @@ -670,6 +719,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.44" @@ -713,6 +774,16 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -779,6 +850,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.4" @@ -795,6 +872,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "compression-codecs" version = "0.4.37" @@ -1019,12 +1109,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -1032,6 +1116,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", + "rand_core 0.6.4", "typenum", ] @@ -1044,6 +1129,15 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -1093,27 +1187,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_more" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "unicode-xid", -] - [[package]] name = "des" version = "0.7.0" @@ -1121,7 +1194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac41dd49fb554432020d52c875fc290e110113f864c6b1b525cd62c7e7747a5d" dependencies = [ "byteorder", - "cipher", + "cipher 0.3.0", "opaque-debug", ] @@ -1196,6 +1269,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + [[package]] name = "dunce" version = "1.0.5" @@ -1502,6 +1581,31 @@ dependencies = [ "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -1563,10 +1667,14 @@ dependencies = [ ] [[package]] -name = "gimli" -version = "0.32.3" +name = "ghash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] [[package]] name = "git-version" @@ -1686,6 +1794,12 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.10.0" @@ -1758,7 +1872,7 @@ dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "triomphe", ] @@ -1870,7 +1984,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "system-configuration", "tokio", "tower-service", @@ -2012,16 +2126,31 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "inventory" version = "0.3.22" @@ -2031,17 +2160,6 @@ dependencies = [ "rustversion", ] -[[package]] -name = "io-uring" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d09b98f7eace8982db770e4408e7470b028ce513ac28fecdc6bf4c30fe92b62" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - [[package]] name = "ipnet" version = "2.12.0" @@ -2082,15 +2200,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2109,6 +2218,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -2212,12 +2330,6 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lazy_static" version = "1.5.0" @@ -2376,7 +2488,7 @@ version = "3.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c42f95f9d296f2dcb50665f507ed5a68a171453142663ce44d77a4eb217b053" dependencies = [ - "aes", + "aes 0.7.5", "base64 0.21.7", "block-modes", "crc-any", @@ -2388,61 +2500,26 @@ dependencies = [ ] [[package]] -name = "malachite" -version = "0.4.22" +name = "manyhow" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", + "manyhow-macros", + "proc-macro2", + "quote", + "syn 3.0.7", ] [[package]] -name = "malachite-base" -version = "0.4.22" +name = "manyhow-macros" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", + "proc-macro-utils", + "proc-macro2", + "quote", ] [[package]] @@ -2575,9 +2652,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -2949,6 +3026,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap", +] + [[package]] name = "owo-colors" version = "4.3.0" @@ -2984,12 +3070,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pem" version = "3.0.6" @@ -3173,6 +3253,18 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + [[package]] name = "postgres-native-tls" version = "0.5.1" @@ -3286,6 +3378,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -3401,9 +3504,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror 2.0.18", "tokio", "tracing", @@ -3422,7 +3525,7 @@ dependencies = [ "lru-slab", "rand 0.9.5", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3441,7 +3544,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -3455,6 +3558,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.7", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -3835,6 +3960,106 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "char_str", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror 2.0.18", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.6", +] + [[package]] name = "rust_decimal" version = "1.40.0" @@ -3851,18 +4076,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "rustc-demangle" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -3971,63 +4184,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.22" @@ -4040,6 +4196,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + [[package]] name = "same-file" version = "1.0.6" @@ -4409,16 +4571,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -4680,15 +4832,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4830,7 +4982,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "siphasher 0.3.11", "swc_atoms", @@ -4852,7 +5004,7 @@ dependencies = [ "num-bigint", "once_cell", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "string_enum", "swc_atoms", "swc_common", @@ -4871,7 +5023,7 @@ dependencies = [ "either", "num-bigint", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "seq-macro", "serde", "smallvec", @@ -4974,6 +5126,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d62a2e0561533f2ca2561d0cf27fd9fedb640a1bf2616ff5d5c80d99017faadc" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5102,6 +5265,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -5193,15 +5362,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.2" @@ -5229,34 +5389,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.1" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.5.10", + "socket2", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.7", ] [[package]] @@ -5289,7 +5446,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.10.1", - "socket2 0.6.3", + "socket2", "tokio", "tokio-util", "whoami 2.1.0", @@ -5325,6 +5482,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -5663,17 +5821,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -5686,17 +5833,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -5776,25 +5912,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] -name = "unicode_names2" -version = "1.3.0" +name = "universal-hash" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.6", + "crypto-common 0.1.7", + "subtle", ] [[package]] @@ -6216,8 +6340,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.826.0" +version = "1.830.0" dependencies = [ + "aes-gcm", "aho-corasick", "anyhow", "arc-swap", @@ -6245,6 +6370,7 @@ dependencies = [ "git-version", "globset", "hex", + "hkdf", "hmac 0.12.1", "hyper", "indexmap", @@ -6260,12 +6386,14 @@ dependencies = [ "pin-project-lite", "pkcs1", "postgres-native-tls", + "postgres-protocol", "quick_cache", "rand 0.9.5", "regex", "reqwest", "reqwest-middleware", "reqwest-retry", + "ryu-js", "semver", "serde", "serde_json", @@ -6302,7 +6430,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.826.0" +version = "1.830.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6442,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.826.0" +version = "1.830.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6451,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6475,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6487,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6499,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6511,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6522,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,30 +6533,30 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] [[package]] name = "windmill-parser-py-asset" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] [[package]] name = "windmill-parser-py-imports" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", @@ -6438,7 +6566,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -6450,7 +6579,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6591,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6605,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6622,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6635,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6647,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6665,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,11 +6681,12 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9", @@ -6568,7 +6698,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6730,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6744,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.826.0" +version = "1.830.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index ba69f6e380..8aebf20597 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.826.0" +version = "1.830.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index 81104d6764..fd953213a4 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -63,12 +63,14 @@ use windmill_common::{ SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, }, + hub_resource_types, scripts::ScriptLang, stats_oss::schedule_stats, triggers::TriggerKind, @@ -121,7 +123,8 @@ use windmill_worker::{ BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, + PY314_CACHE_DIR, PY315_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, + TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -401,111 +404,16 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { Ok(()) } -/// Raw resource type from hub API (schema is a JSON string) -#[derive(serde::Deserialize)] -struct HubResourceTypeRaw { - pub id: i64, - pub name: String, - pub schema: Option, - pub app: String, - pub description: Option, - /// Absent from hubs predating the column, and from caches written before it. - #[serde(default)] - pub format_extension: Option, - /// Doubly optional, so a hub predating the field (no key) is told apart from a type the - /// hub leaves unnamed (null). - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - pub display_name: Option>, -} - -/// Processed resource type with parsed schema -#[derive(serde::Deserialize, serde::Serialize, Clone)] -pub struct HubResourceType { - pub id: i64, - pub name: String, - pub schema: Option, - pub app: String, - pub description: Option, - /// Doubly optional on purpose. A cache written before this column has no key at - /// all (`None`) and must leave the stored extension alone; one written since - /// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely - /// dropping it and must clear. A single `Option` conflates the two, and picking - /// either meaning breaks the other — as does plain serde, which folds `null` - /// into the outer `None`, hence the wrapping deserializer. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option", - skip_serializing_if = "Option::is_none" - )] - pub format_extension: Option>, - /// Doubly optional like `format_extension`: a cache written before the field leaves the - /// stored name alone, while a null from the hub clears it. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option", - skip_serializing_if = "Option::is_none" - )] - pub display_name: Option>, -} - -const HUB_RT_CACHE_FILE: &str = "resource_types.json"; - async fn cache_hub_resource_types() -> anyhow::Result<()> { println!("Caching resource types from hub..."); - let response = HTTP_CLIENT - .get(format!("{}/resource_types/list", DEFAULT_HUB_BASE_URL)) - .header("Accept", "application/json") - .send() - .await - .with_context(|| "Failed to fetch resource types from hub")?; - - if !response.status().is_success() { - anyhow::bail!( - "Failed to fetch resource types from hub: {}", - response.status() - ); - } - - let raw_types: Vec = response - .json::>() - .await - .with_context(|| "Failed to parse resource types from hub")?; - - // Parse schema strings into JSON values - let resource_types: Vec = raw_types - .into_iter() - .filter_map(|rt| { - let schema = match rt.schema { - Some(s) => match serde_json::from_str(&s) { - Ok(v) => Some(v), - Err(e) => { - println!("Warning: failed to parse schema for {}: {}", rt.name, e); - return None; - } - }, - None => None, - }; - Some(HubResourceType { - id: rt.id, - name: rt.name, - schema, - app: rt.app, - description: rt.description, - format_extension: Some(rt.format_extension), - display_name: rt.display_name, - }) - }) - .collect(); + let resource_types = hub_resource_types::fetch(DEFAULT_HUB_BASE_URL, None, None).await?; println!("Fetched {} resource types from hub", resource_types.len()); create_dir_all(&*HUB_RT_CACHE_DIR)?; - let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = hub_resource_types::cache_path(); let content = serde_json::to_string_pretty(&resource_types) .with_context(|| "Failed to serialize resource types")?; @@ -516,147 +424,6 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { Ok(()) } -pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyhow::Result<()> { - let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); - - if tokio::fs::metadata(&cache_path).await.is_err() { - tracing::info!( - "No cached resource types found at {}, skipping sync", - cache_path - ); - return Ok(()); - } - - tracing::info!("Syncing cached resource types to admins workspace..."); - - let content = tokio::fs::read_to_string(&cache_path) - .await - .with_context(|| format!("Failed to read cache file from {}", cache_path))?; - - let cached_types: Vec = - serde_json::from_str(&content).with_context(|| "Failed to parse cached resource types")?; - - tracing::info!("Found {} cached resource types", cached_types.len()); - - // Get existing resource types in admins workspace. `format_extension` is part of - // the comparison below, so a type whose only change is gaining or losing it is - // not mistaken for unchanged; `is_fileset` decides whether it may take one. - let existing_types: Vec<( - String, - Option, - Option, - Option, - bool, - Option, - )> = sqlx::query_as( - "SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'", - ) - .fetch_all(db) - .await - .with_context(|| "Failed to fetch existing resource types")?; - - let existing_map: std::collections::HashMap< - String, - ( - Option, - Option, - Option, - bool, - Option, - ), - > = existing_types - .into_iter() - .map( - |(name, schema, desc, format_extension, is_fileset, display_name)| { - ( - name, - (schema, desc, format_extension, is_fileset, display_name), - ) - }, - ) - .collect(); - - let mut synced_count = 0; - let mut skipped_count = 0; - - for rt in cached_types { - let existing = existing_map.get(&rt.name); - let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false); - let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone()); - let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone()); - // A fileset is a set of files, so it cannot also be one file. Create, update - // and the manual sync all reject the pair; this writer would otherwise - // persist it onto a same-named local fileset. - // - // A cache with no key at all leaves the stored value alone, so the target is - // what is already there — which is also what makes the comparison below - // agree with the write instead of re-upserting the row on every boot. - let format_extension = if is_fileset { - None - } else { - match &rt.format_extension { - Some(from_cache) => from_cache.clone(), - None => stored_extension.clone(), - } - }; - // No key in the cache leaves the stored name alone, as for the extension. So does a name - // too long for the column: one bad entry must not fail the upsert and end the sync. - let display_name = match &rt.display_name { - Some(Some(name)) if name.chars().count() > 100 => { - tracing::warn!( - "Ignoring the display_name of resource type {}: longer than 100 characters", - rt.name - ); - stored_display_name.clone() - } - Some(from_cache) => from_cache.clone(), - None => stored_display_name.clone(), - }; - - if let Some((existing_schema, existing_desc, _, _, _)) = existing { - if existing_schema == &rt.schema - && existing_desc == &rt.description - && stored_extension == format_extension - && stored_display_name == display_name - { - skipped_count += 1; - continue; - } - } - - // Insert or update resource type - sqlx::query( - // `format_extension` is resolved above rather than coalesced here: a - // COALESCE could never clear one, so a hub that dropped an extension - // would leave the stale value behind forever. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) - VALUES ('admins', $1, $2, $3, $4, $5, now()) - ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, - format_extension = EXCLUDED.format_extension, - display_name = EXCLUDED.display_name, edited_at = now()", - ) - .bind(&rt.name) - .bind(&rt.schema) - .bind(&rt.description) - .bind(&format_extension) - .bind(&display_name) - .execute(db) - .await - .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; - - synced_count += 1; - } - - tracing::info!( - "Synced {} resource types to admins workspace ({} skipped as unchanged)", - synced_count, - skipped_count - ); - - Ok(()) -} - fn print_help() { println!("Windmill - a fast, open-source workflow engine and job runner."); println!(); @@ -670,6 +437,8 @@ fn print_help() { println!(" cache-rt Pre-cache hub resource types"); println!(" sync-config Sync instance config from a YAML file to the database"); println!(" operator Run the Kubernetes operator (watches a ConfigMap)"); + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + println!(" oidc-signer (ee) Check an external OIDC signer, or run a file-backed one for tests"); println!(); println!("Environment variables (name = default):"); println!(" DATABASE_URL = The Postgres database url."); @@ -696,7 +465,7 @@ fn print_help() { println!(" LICENSE_KEY = None (EE only) Enterprise license key (workers require valid key)"); println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup"); println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool"); - println!(" SYNC_CACHED_RT = false Sync cached resource types to admins workspace on server start"); + println!(" SYNC_CACHED_RT = false Sync the image's cached resource types to the admins workspace on server start"); println!(" HUB_BASE_URL = https://hub.windmill.dev Hub to fetch scripts from in `cache` mode (server/worker use the DB setting instead)"); println!(); println!("Notes:"); @@ -704,6 +473,38 @@ fn print_help() { println!("- At startup, Windmill logs currently set configuration keys for visibility."); } +async fn otel_environment(conn: &Connection) -> String { + if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") { + environment + } else { + load_base_url(conn) + .await + .unwrap_or_else(|_| "local".to_string()) + .trim_start_matches("https://") + .trim_start_matches("http://") + .split(".") + .next() + .unwrap_or_else(|| "local") + .to_string() + } +} + +/// Tracing for a subcommand that writes instance config. It never runs the server startup +/// that loads the OTel settings, and its audit events are exported under them. +/// +/// The subscriber it installs depends on those settings, so `early_logs` is what prints +/// the events of connecting and loading them; it gives way once the real one is in. +async fn init_declarative_writer_tracing( + db: &windmill_common::DB, + service: &str, + early_logs: tracing::subscriber::DefaultGuard, +) { + load_otel(db).await; + let environment = otel_environment(&Connection::Sql(db.clone())).await; + drop(early_logs); + windmill_common::tracing_init::initialize_subcommand_tracing(service, &HOSTNAME, &environment); +} + async fn windmill_main() -> anyhow::Result<()> { let (killpill_tx, mut killpill_rx) = KillpillSender::new(2); let mut monitor_killpill_rx = killpill_tx.subscribe(); @@ -807,7 +608,6 @@ async fn windmill_main() -> anyhow::Result<()> { return Ok(()); } "sync-config" => { - tracing_subscriber::fmt::init(); let path = std::env::args().nth(2).unwrap_or_else(|| { eprintln!("Usage: windmill sync-config "); std::process::exit(1); @@ -821,15 +621,40 @@ async fn windmill_main() -> anyhow::Result<()> { windmill_common::instance_config::resolve_env_refs(&mut config.global_settings) .map_err(|var| anyhow::anyhow!("environment variable '{var}' not found"))?; + let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish()); tracing::info!("Connecting to database..."); let db = crate::db_connect::initial_connection().await?; - config.sync_to_db(&db).await?; + init_declarative_writer_tracing(&db, "windmill-sync-config", early_logs).await; + + let author = windmill_audit::instance_config::declarative_sync_author(); + let synced = config + .sync_to_db( + &db, + &windmill_audit::instance_config::InstanceConfigAuditor { + db: &db, + author: &author, + source: Some( + windmill_audit::instance_config::DeclarativeSource::SyncConfig, + ), + }, + ) + .await; + // Also on failure: the settings written before it are already audited, and the + // process exits before the exporter's next scheduled batch. + windmill_common::otel_oss::flush_otel_logs().await; + synced?; tracing::info!("Synced instance config from {path}"); return Ok(()); } + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + "oidc-signer" => { + tracing_subscriber::fmt::init(); + windmill_api::oidc_signer_tools_ee::run_cli(std::env::args().skip(2).collect()).await?; + return Ok(()); + } #[cfg(feature = "operator")] "operator" => { - tracing_subscriber::fmt::init(); + let early_logs = tracing::subscriber::set_default(tracing_subscriber::fmt().finish()); tracing::info!("Starting Windmill Kubernetes operator..."); tracing::info!("Connecting to database..."); @@ -842,6 +667,7 @@ async fn windmill_main() -> anyhow::Result<()> { operator_killpill_rx, ) .await?; + init_declarative_writer_tracing(&db, "windmill-operator", early_logs).await; #[cfg(all(feature = "enterprise", feature = "private"))] tokio::spawn(async move { @@ -968,19 +794,7 @@ async fn windmill_main() -> anyhow::Result<()> { (Connection::Sql(db), None, None) }; - let environment = if let Ok(environment) = std::env::var("OTEL_ENVIRONMENT") { - environment - } else { - load_base_url(&conn) - .await - .unwrap_or_else(|_| "local".to_string()) - .trim_start_matches("https://") - .trim_start_matches("http://") - .split(".") - .next() - .unwrap_or_else(|| "local") - .to_string() - }; + let environment = otel_environment(&conn).await; let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment); @@ -1015,8 +829,10 @@ async fn windmill_main() -> anyhow::Result<()> { .map(|v| v.to_lowercase() == "true" || v == "1") .unwrap_or(false) { - if let Err(e) = sync_cached_resource_types(db).await { - tracing::warn!("Failed to sync cached resource types: {:#}", e); + match hub_resource_types::sync_from_image_cache(db).await { + Ok(Some(outcome)) => tracing::info!("{}", outcome.summary()), + Ok(None) => {} + Err(e) => tracing::warn!("Failed to sync cached resource types: {:#}", e), } } } @@ -1234,6 +1050,15 @@ Windmill Community Edition {GIT_VERSION} // empty script cache, refilled from the database. windmill_common::cache::script::clear(); + // Before the first load: it can sign a token for instance object storage. + #[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] + windmill_common::oidc_ee::init_oidc_signing( + &conn, + server_mode, + killpill_rx.resubscribe(), + ) + .await?; + initial_load( &conn, killpill_tx.clone(), @@ -1748,6 +1573,7 @@ Windmill Community Edition {GIT_VERSION} if server_mode { if let Some(db) = conn.as_sql() { schedule_stats(&db, &HTTP_CLIENT).await; + hub_resource_types::spawn_daily_sync(db.clone(), killpill_tx.subscribe()); } } @@ -2322,6 +2148,12 @@ async fn process_notify_event( RESTART_COORDINATION_SETTING => { // Internal coordination key for staggered restarts, no action needed } + SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING => { + // Read by the daily resource type sync each time it runs + } + "oidc_external_signer" => { + // Written by servers that sign with an external signer, read when signing + } "plain_emails_telemetry" => { let enabled = sqlx::query_scalar!( "SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'" @@ -2407,6 +2239,8 @@ pub async fn run_workers( &*PY311_CACHE_DIR, &*PY312_CACHE_DIR, &*PY313_CACHE_DIR, + &*PY314_CACHE_DIR, + &*PY315_CACHE_DIR, &*BUN_BUNDLE_CACHE_DIR, &*GO_CACHE_DIR, &*GO_BIN_CACHE_DIR, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 677ecd11a2..23ab2eaed1 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -427,6 +427,8 @@ pub async fn initial_load( native_mode, // an agent worker never reads its group's config, only its token object_store_cache_config: None, + // nor whether it is paused: the server refuses it jobs instead + paused: false, })); } } @@ -4134,6 +4136,20 @@ pub async fn monitor_db( } _ => {} } + match windmill_common::runnable_job_stats::cleanup_old_runnable_job_stats( + db, + windmill_common::runnable_job_stats::RETENTION_DAYS, + ) + .await + { + Ok(count) if count > 0 => { + tracing::info!("Deleted {} old runnable job stats rows", count); + } + Err(e) => { + tracing::error!("Error cleaning up runnable job stats: {:?}", e); + } + _ => {} + } } } }; @@ -4433,6 +4449,42 @@ pub async fn monitor_db( } }; + // Drop the data table role set logins nobody has connected with for a month. Every ~40 min, + // spawned for the same reason as the credential maintenance above: it opens a connection per + // managed database for each login it drops. + let datatable_role_set_sweep_f = async { + if server_mode && iteration.is_some() && iteration.as_ref().unwrap().should_run(240) { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { + if let Err(e) = + windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await + { + tracing::error!("Error sweeping unused data table role set logins: {e}"); + } + }); + } + } + }; + + // Re-encrypt the legacy-format secret variables once new values are written in AES-256-GCM. + // Every ~5 min until one pass has gone through every workspace; spawned so a large instance + // does not hold up the join below. + let legacy_secret_upgrade_f = async { + if server_mode + && iteration.is_some() + && iteration.as_ref().unwrap().should_run(30) + && windmill_common::variables::legacy_secret_upgrade_pending() + { + if let Some(db) = conn.as_sql() { + let db = db.clone(); + tokio::spawn(async move { + windmill_common::variables::upgrade_legacy_secret_variables(&db).await + }); + } + } + }; + // Delete the AI session backups older than their workspace's retention. Every ~40 min // (240 iterations at the default 10 s, the most a u8 `should_run` counts): the retention // counts in days. Spawned for the same reason as the credential maintenance above, a @@ -4504,7 +4556,9 @@ pub async fn monitor_db( cleanup_scheduled_job_deletions_f, git_auto_pull_f, git_credential_maintenance_f, + datatable_role_set_sweep_f, ai_session_retention_f, + legacy_secret_upgrade_f, pipeline_freshness_watchdog_f, reconcile_unarmed_schedules_f, stranded_jobs_f, @@ -5689,8 +5743,13 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b .as_ref() .is_some_and(|dws| !dws.is_empty()); + // Pausing or resuming must not restart the workers, dedicated ones included: a reload + // whose only difference is the flag skips every restart below. + let only_pause_changed = wc.paused != config.paused + && WorkerConfig { paused: wc.paused, ..config.clone() } == **wc; + if **wc != config || has_dedicated { - if kill_if_change { + if kill_if_change && !only_pause_changed { if has_dedicated || wc.dedicated_worker != config.dedicated_worker || wc.dedicated_workers != config.dedicated_workers diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 2a518a7cc8..ac07dc06d6 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -180,6 +180,7 @@ resource_type: workspace_id(char), name(char), schema(jsonb), description(text), FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) +runnable_job_stats: hour(bigint), workspace_id(char), runnable_path(char), worker_group(text), job_count(int), total_duration_ms(bigint), max_memory_peak(int), sum_memory_peak(bigint), memory_sample_count(int), total_cpu_ms(bigint) runnable_settings: hash(bigint), debouncing_settings(bigint), concurrency_settings(bigint) schedule: workspace_id(char), path(char), edited_by(char), edited_at(ts), schedule(char), enabled(bool), script_path(char), args(jsonb), extra_perms(jsonb), is_flow(bool), email(char), error(text), timezone(char), on_failure(char), on_recovery(char), on_failure_times(int), on_failure_exact(bool), on_failure_extra_args(jsonb), on_recovery_times(int), on_recovery_extra_args(jsonb), ws_error_handler_muted(bool), retry(jsonb), summary(char), no_flow_overlap(bool), tag(char), paused_until(ts), on_success(char), on_success_extra_args(jsonb), cron_version(text), description(text), dynamic_skip(char), labels(text[]) FK: (workspace_id) -> workspace(id) @@ -219,7 +220,7 @@ websocket_trigger: path(char), url(char), script_path(char), is_flow(bool), work windmill_migrations: name(text), created_at(ts) worker_group_job_stats: hour(bigint), worker_group(text), script_lang(char), workspace_id(char), job_count(int), total_duration_ms(bigint) FK: (workspace_id) -> workspace(id) -worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]) +worker_ping: worker(char), worker_instance(char), ping_at(ts), started_at(ts), ip(char), jobs_executed(int), custom_tags(text[]), worker_group(char), dedicated_worker(char), wm_version(char), current_job_id(uuid), current_job_workspace_id(char), vcpus(bigint), memory(bigint), occupancy_rate(float), memory_usage(bigint), wm_memory_usage(bigint), occupancy_rate_15s(float), occupancy_rate_5m(float), occupancy_rate_30m(float), job_isolation(text), dedicated_workers(text[]), draining(bool) workspace: id(char), name(char), owner(char), deleted(bool), premium(bool), parent_workspace_id(char), is_dev_workspace(bool) FK: (parent_workspace_id) -> workspace(id) workspace_dependencies: id(bigint), name(char), content(text), language(script_lang), description(text), archived(bool), workspace_id(char), created_at(ts) diff --git a/backend/tests/ai_decision.rs b/backend/tests/ai_decision.rs index 04b310f2c1..d6359c1cea 100644 --- a/backend/tests/ai_decision.rs +++ b/backend/tests/ai_decision.rs @@ -117,7 +117,7 @@ async fn test_ai_decision_answers_route_a_following_branch( "expr": "results.d.output.intent.choice === 'refund'", "modules": [{"id": "r", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(choice){ return 'handled ' + choice }", "input_transforms": {"choice": { "type": "javascript", diff --git a/backend/tests/app_ctx_inputs.rs b/backend/tests/app_ctx_inputs.rs new file mode 100644 index 0000000000..cf386adc89 --- /dev/null +++ b/backend/tests/app_ctx_inputs.rs @@ -0,0 +1,88 @@ +//! A context input a raw app runnable declares (`fields. = { type: "ctx", .. }`) is +//! filled by the server from the viewer's session. Whatever the request sends under that +//! name is discarded: a runnable filters rows on it, so a viewer who could set it would +//! read someone else's. +//! +//! Users from the `base` fixture: +//! test-user (admin, token SECRET_TOKEN) +//! test-user-2 (developer, token SECRET_TOKEN_2) + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use windmill_test_utils::*; + +const ADMIN_TOKEN: &str = "SECRET_TOKEN"; +const VIEWER_TOKEN: &str = "SECRET_TOKEN_2"; +const APP_PATH: &str = "u/test-user/ctx_app"; +const CONTENT: &str = "export function main(who: string) { return who }"; +// sha256 of CONTENT, the key an inline runnable is pinned under. +const CONTENT_SHA: &str = "6e198199621d57e4fed834510cb0c2692274aa4cbe976e1704fab2806be9d744"; + +fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuilder { + builder.header("Authorization", format!("Bearer {}", token)) +} + +#[sqlx::test(fixtures("base"))] +async fn a_declared_ctx_input_ignores_the_value_the_viewer_sends( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let ws = format!("http://localhost:{port}/api/w/test-workspace"); + let client = reqwest::Client::new(); + + let resp = authed(client.post(format!("{ws}/apps/create")), ADMIN_TOKEN) + .json(&json!({ + "path": APP_PATH, + "summary": "ctx inputs", + "value": { + "runnables": { + "whoami": { + "type": "inline", + "inlineScript": { "language": "bun", "content": CONTENT }, + "fields": { "who": { "type": "ctx", "ctx": "username" } } + } + } + }, + // The policy says nothing of context inputs: the server reads the + // declaration from the deployed runnable. + "policy": { + "execution_mode": "anonymous", + "triggerables_v2": { + format!("whoami:rawscript/{CONTENT_SHA}"): { + "static_inputs": {}, + "one_of_inputs": {} + } + } + } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + + // Both spellings a viewer could send: a forged literal, and no argument at all. + for args in [json!({ "who": "test-user" }), json!({})] { + let resp = authed( + client.post(format!("{ws}/apps_u/execute_component/{APP_PATH}")), + VIEWER_TOKEN, + ) + .json(&json!({ + "component": "whoami", + "args": args, + "raw_code": { "language": "bun", "content": CONTENT } + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let job_id: uuid::Uuid = resp.text().await?.parse()?; + + let queued: serde_json::Value = sqlx::query_scalar("SELECT args FROM v2_job WHERE id = $1") + .bind(job_id) + .fetch_one(&db) + .await?; + assert_eq!(queued["who"], json!("test-user-2"), "sent {args}"); + } + + Ok(()) +} diff --git a/backend/tests/debounce_e2e.rs b/backend/tests/debounce_e2e.rs index 9a2351f135..709629d825 100644 --- a/backend/tests/debounce_e2e.rs +++ b/backend/tests/debounce_e2e.rs @@ -33,7 +33,7 @@ mod debounce_e2e { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main(items: any[]) { return items }", "input_transforms": { "items": { "type": "javascript", "expr": "flow_input.items" } diff --git a/backend/tests/drafts_delete_for_user.rs b/backend/tests/drafts_delete_for_user.rs new file mode 100644 index 0000000000..9c48f8828c --- /dev/null +++ b/backend/tests/drafts_delete_for_user.rs @@ -0,0 +1,85 @@ +//! Only a workspace admin can delete a draft that belongs to another user. +//! +//! A discard through `drafts/update` is scoped to the caller's own row, so it +//! never reaches someone else's draft. `drafts/delete_for_user` is the admin +//! route for that, and it must stay closed to everyone else. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_only_admin_deletes_another_users_draft(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/drafts"); + let client = reqwest::Client::new(); + let path = "script/u/test-user-2/theirs"; + + let save = |token: &'static str| { + client + .post(format!("{base}/update/{path}")) + .header("Authorization", format!("Bearer {token}")) + .json(&json!({ "value": { "path": "u/test-user-2/theirs", "content": "" } })) + .send() + }; + let resp = save("SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let delete = |token: &'static str| { + client + .post(format!( + "{base}/delete_for_user/{path}?username=test-user-2" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let own_draft = || async { + client + .get(format!("{base}/get_own/{path}")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .send() + .await? + .json::() + .await + }; + + let resp = delete("SECRET_TOKEN_3").await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + assert!( + !own_draft().await?.is_null(), + "a non-admin deleted the draft" + ); + + let resp = delete("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert!(own_draft().await?.is_null(), "the draft is still there"); + + let resp = delete("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 404, "{}", resp.text().await?); + + // Without a username the delete takes every draft at the path. + for token in ["SECRET_TOKEN_2", "SECRET_TOKEN"] { + let resp = save(token).await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let delete_all = |token: &'static str| { + client + .post(format!("{base}/delete_for_user/{path}")) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let resp = delete_all("SECRET_TOKEN_3").await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + let resp = delete_all("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let left: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user-2/theirs'", + ) + .fetch_one(&db) + .await?; + assert_eq!(left, 0, "drafts survived a path-wide delete"); + Ok(()) +} diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs index 366c220c62..0d8ff7be9a 100644 --- a/backend/tests/end_user_email.rs +++ b/backend/tests/end_user_email.rs @@ -1,10 +1,8 @@ //! Tests for WM_END_USER_EMAIL environment variable. //! //! These tests verify that WM_END_USER_EMAIL is populated with the authenticated -//! user's email when executing app components. -//! -//! TODO: Add tests for scripts and flows once public execution endpoints are identified. -//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup. +//! user's email when executing app components and when running scripts and flows directly. +//! Only apps support non-workspace-member execution via OptAuthed + token lookup. use serde_json::json; use sqlx::{Pool, Postgres}; @@ -27,40 +25,6 @@ fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuil builder.header("Authorization", format!("Bearer {}", token)) } -// TODO: Script tests - need to identify public execution endpoints for non-workspace-members -// async fn run_script(port: u16, token: &str) -> anyhow::Result { -// let url = format!( -// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email", -// port -// ); -// let resp = authed(client().post(&url), token) -// .json(&json!({})) -// .send() -// .await?; -// if !resp.status().is_success() { -// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?); -// } -// Ok(resp.json::().await? -// .as_str().unwrap_or("").to_string()) -// } - -// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members -// async fn run_flow(port: u16, token: &str) -> anyhow::Result { -// let url = format!( -// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow", -// port -// ); -// let resp = authed(client().post(&url), token) -// .json(&json!({})) -// .send() -// .await?; -// if !resp.status().is_success() { -// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?); -// } -// Ok(resp.json::().await? -// .as_str().unwrap_or("").to_string()) -// } - /// Create an app with inline script via API async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { let url = format!("http://localhost:{}/api/w/test-workspace/apps/create", port); @@ -74,8 +38,8 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result< "subgrids": {}, "hiddenInlineScripts": [{ "name": "get_email", - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "path": "f/test/email_app/get_email" }] }, @@ -89,7 +53,7 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result< "one_of_inputs": {} }, // SHA256 hash of raw_code content for anonymous execution - "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": { "static_inputs": {}, "one_of_inputs": {} } @@ -120,8 +84,8 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res "css": "", "inlineScripts": [{ "name": "get_email", - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" }] }, "policy": { @@ -134,7 +98,7 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res "one_of_inputs": {} }, // SHA256 hash of raw_code content for anonymous execution - "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": { "static_inputs": {}, "one_of_inputs": {} } @@ -233,8 +197,8 @@ async fn run_app_inline_script( "args": {}, "component": "get_email", "raw_code": { - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "path": format!("{}/get_email", app_path) } }); @@ -270,8 +234,8 @@ async fn run_raw_app_inline_script( "args": {}, "component": "get_email", "raw_code": { - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" } }); if force_viewer { @@ -312,41 +276,111 @@ async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Re anyhow::bail!("timeout waiting for job result") } -// TODO: Script tests - need to identify public execution endpoints for non-workspace-members -// #[cfg(feature = "deno_core")] -// #[sqlx::test(fixtures("base", "end_user_email"))] -// async fn test_script_wm_end_user_email(db: Pool) -> anyhow::Result<()> { -// initialize_tracing().await; -// set_jwt_secret().await; -// let server = ApiServer::start(db.clone()).await?; -// let port = server.addr.port(); -// -// in_test_worker(Connection::Sql(db.clone()), async move { -// let result = run_script(port, SAME_WS_TOKEN).await?; -// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); -// Ok::<(), anyhow::Error>(()) -// }, port).await?; -// -// Ok(()) -// } +async fn run_direct(port: u16, token: &str, route: &str) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/jobs/run/{}", + port, route + ); + let resp = authed(client().post(&url), token) + .json(&json!({})) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!( + "run {route} failed: {} - {}", + resp.status(), + resp.text().await? + ); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} -// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members -// #[cfg(feature = "deno_core")] -// #[sqlx::test(fixtures("base", "end_user_email"))] -// async fn test_flow_wm_end_user_email(db: Pool) -> anyhow::Result<()> { -// initialize_tracing().await; -// set_jwt_secret().await; -// let server = ApiServer::start(db.clone()).await?; -// let port = server.addr.port(); -// -// in_test_worker(Connection::Sql(db.clone()), async move { -// let result = run_flow(port, SAME_WS_TOKEN).await?; -// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); -// Ok::<(), anyhow::Error>(()) -// }, port).await?; -// -// Ok(()) -// } +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_direct_run_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + in_test_worker( + Connection::Sql(db.clone()), + async move { + let result = run_direct(port, SAME_WS_TOKEN, "p/f/test/get_end_user_email").await?; + assert_eq!(result, SAME_WS_EMAIL, "a script run should see its caller"); + + let result = + run_direct(port, SAME_WS_TOKEN, "f/f/test/get_end_user_email_flow").await?; + assert_eq!( + result, SAME_WS_EMAIL, + "every flow step should see the flow's caller" + ); + + Ok::<(), anyhow::Error>(()) + }, + port, + ) + .await?; + + Ok(()) +} + +/// A job's `WM_TOKEN` names the identity the job runs as, which differs from its end user when +/// the job runs on behalf of another account. A run it starts must keep the job's end user. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_job_token_run_keeps_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let calling_job = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email) + VALUES ($1, 'test2@windmill.dev', 'test-user-2', false, false, '{}', '{}', 'test-workspace', $2)", + ) + .bind(calling_job) + .bind(SAME_WS_EMAIL) + .execute(&db) + .await?; + let wm_token = windmill_common::auth::create_jwt_token( + windmill_common::db::Authed { + email: "test2@windmill.dev".to_string(), + username: "test-user-2".to_string(), + is_admin: false, + is_operator: false, + groups: vec!["all".to_string()], + folders: vec![], + scopes: None, + token_prefix: None, + }, + "test-workspace", + 3600, + Some(calling_job), + None, + None, + None, + ) + .await?; + + in_test_worker( + Connection::Sql(db.clone()), + async move { + let result = run_direct(port, &wm_token, "p/f/test/get_end_user_email").await?; + assert_eq!( + result, SAME_WS_EMAIL, + "the calling job's end user, not its run-as identity" + ); + Ok::<(), anyhow::Error>(()) + }, + port, + ) + .await?; + + Ok(()) +} #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "end_user_email"))] diff --git a/backend/tests/error_handler.rs b/backend/tests/error_handler.rs index 4f58685221..2e10006d2f 100644 --- a/backend/tests/error_handler.rs +++ b/backend/tests/error_handler.rs @@ -104,7 +104,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '') + VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\n//bun.lock\n') "#, error_handler_code ) @@ -118,7 +118,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '') + VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\n//bun.lock\n') "#, failing_script_hash, failing_script_code @@ -155,7 +155,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -234,7 +234,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, ) .execute(&db) @@ -245,7 +245,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted) - VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true) + VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n', true) "#, failing_script_hash, ) @@ -280,7 +280,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -331,7 +331,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, ) .execute(&db) @@ -342,7 +342,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, success_script_hash, ) @@ -377,7 +377,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its failure must reach the workspace error handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_error_handler_triggered_on_hub_script_failure( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000002"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/fail"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(!completed_job.success, "Hub script should have failed"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace error handler should handle the failed hub script" + ); + + Ok(()) +} diff --git a/backend/tests/fixtures/end_user_email.sql b/backend/tests/fixtures/end_user_email.sql index 596d34e3e3..3b376525a4 100644 --- a/backend/tests/fixtures/end_user_email.sql +++ b/backend/tests/fixtures/end_user_email.sql @@ -40,9 +40,9 @@ VALUES (encode(sha256('NO_WS_TOKEN'::bytea), 'hex'), 'NO_WS_TOKE', 'NO_WS_TOKEN' INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms) VALUES ( 'test-workspace', 'test-user', - 'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }', + 'export function main() { return process.env.WM_END_USER_EMAIL || ""; }', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script', + 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'bun', E'{}\n//bun.lock\n', 'script', '{"g/all": true}' ); @@ -51,7 +51,7 @@ INSERT INTO flow (workspace_id, summary, description, path, versions, schema, va VALUES ( 'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}', 'test-user', '{"g/all": true}' ); @@ -60,6 +60,6 @@ INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) VALUES ( 900002, 'test-workspace', 'f/test/get_end_user_email_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}', 'test-user' ); diff --git a/backend/tests/fixtures/hello.sql b/backend/tests/fixtures/hello.sql index 962f6441d9..4c40ac3290 100644 --- a/backend/tests/fixtures/hello.sql +++ b/backend/tests/fixtures/hello.sql @@ -11,7 +11,7 @@ export function main(world: string) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', '', '', -'f/system/hello', 123412, 'deno', ''); +'f/system/hello', 123412, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -28,7 +28,7 @@ export async function main(foo: string, bar: string) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', '', '', -'f/system/hello_with_preprocessor', 123413, 'deno', ''); +'f/system/hello_with_preprocessor', 123413, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -140,7 +140,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/system/hello_with_nodes_flow', '{1443253234253454}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', 'system' ); @@ -149,7 +149,7 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/system/hello_with_nodes_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', 'system' ); @@ -160,7 +160,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/system/hello_with_preprocessor', '{1443253234253456}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', -E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', 'system' ); @@ -169,6 +169,6 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/system/hello_with_preprocessor', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', -E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', 'system' ); diff --git a/backend/tests/fixtures/jobs_read_auth.sql b/backend/tests/fixtures/jobs_read_auth.sql index e32b777a74..e175a9c065 100644 --- a/backend/tests/fixtures/jobs_read_auth.sql +++ b/backend/tests/fixtures/jobs_read_auth.sql @@ -98,6 +98,32 @@ INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, resu ('18181818-1818-1818-1818-181818181818', 'test-workspace', 1000, 'success'::job_status, '{"agent": "AGENT_CHAT_RESULT"}'); +-- A lock job for flow `f/shared/flow1`, as `wmill generate-metadata` queues it, and the +-- token CI would lock with. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('RUN_SCOPED_DEPS_TOKEN'::bytea), 'hex'), 'RUN_DEPS_', 'RUN_SCOPED_DEPS_TOKEN', + 'test2@windmill.dev', 'lock token', false, + ARRAY['jobs:run:dependencies:f/shared/*'] +); +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, runnable_path, tag, visible_to_owner +) VALUES ( + '19191919-1919-1919-1919-191919191919', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'flowdependencies', 'f/shared/flow1', 'dependency', true +), ( + -- Queued by a broader token with `*` as its path, which the request body allows. + '20202020-2020-2020-2020-202020202020', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'dependencies', '*', 'dependency', true +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('19191919-1919-1919-1919-191919191919', 'test-workspace', 1000, 'success'::job_status, + '{"lock": "DEPS_RESULT"}'), + ('20202020-2020-2020-2020-202020202020', 'test-workspace', 1000, 'success'::job_status, + '{"lock": "WILDCARD_DEPS_RESULT"}'); + -- A token pairing an app scope with a run scope, as someone driving an app's components -- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no -- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach. diff --git a/backend/tests/fixtures/schedule.sql b/backend/tests/fixtures/schedule.sql index 8d851369b0..999aa4770f 100644 --- a/backend/tests/fixtures/schedule.sql +++ b/backend/tests/fixtures/schedule.sql @@ -13,7 +13,7 @@ export async function main(fail: boolean = true) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}', '', '', -'f/system/failing_script', 12349, 'deno', ''); +'f/system/failing_script', 12349, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -26,7 +26,7 @@ export async function main() { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}', '', '', -'f/system/schedule_error_handler', 123410, 'deno', ''); +'f/system/schedule_error_handler', 123410, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -39,7 +39,7 @@ export async function main() { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}', '', '', -'f/system/schedule_recovery_handler', 123411, 'deno', ''); +'f/system/schedule_recovery_handler', 123411, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( 'test-workspace', diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index 3806bb2886..2d4f72f336 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -85,7 +85,7 @@ async fn test_flow_linear_input_transforms(db: Pool) -> anyhow::Result modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: [static_input("x", 10), static_input("y", 5)].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(x: number, y: number) { return {sum: x + y, product: x * y, items: [1, 2, 3, 4, 5]}; @@ -105,7 +105,7 @@ export function main(x: number, y: number) { js_input("filtered", "results.a.items.filter(x => x > 2)"), js_input("from_flow_input", "flow_input.multiplier * results.a.sum"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(total: number, doubled_items: number[], filtered: number[], from_flow_input: number) { return {total, doubled_items, filtered, from_flow_input}; @@ -157,7 +157,7 @@ async fn test_flow_forloop_complex_expressions(db: Pool) -> anyhow::Re "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -199,7 +199,7 @@ export function main() { js_input("index", "flow_input.iter.index"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(user_name: string, user_score: number, bonus: number, index: number) { return {name: user_name, final_score: user_score + bonus, position: index}; @@ -261,7 +261,7 @@ async fn test_flow_branchone_conditions(db: Pool) -> anyhow::Result<() "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {status: "premium", score: 95, items: [1, 2, 3]}; @@ -293,7 +293,7 @@ export function main() { js_input("score_from_a", "results.a.score"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(discount: number, score_from_a: number) { return {branch: "premium_high", discount, score_from_a}; @@ -320,7 +320,7 @@ export function main(discount: number, score_from_a: number) { "premium_low", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {branch: "premium_low", discount: 10}; @@ -344,7 +344,7 @@ export function main() { "default_branch", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {branch: "default", discount: 0}; @@ -398,7 +398,7 @@ async fn test_flow_previous_result_aggregation(db: Pool) -> anyhow::Re "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {value: 10, items: [1, 2, 3]}; @@ -424,7 +424,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev_value: number, prev_items_sum: number) { return {value: prev_value * 2, sum: prev_items_sum}; @@ -452,7 +452,7 @@ export function main(prev_value: number, prev_items_sum: number) { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(a_value: number, b_value: number, b_sum: number, combined: number) { return {a_value, b_value, b_sum, combined}; @@ -504,7 +504,7 @@ async fn test_flow_nested_complexity(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -541,7 +541,7 @@ export function main() { js_input("values", "results.data.base_values.map(v => v * flow_input.iter.value.multiplier)"), js_input("sum", "results.data.base_values.reduce((a, b) => a + b, 0) * flow_input.iter.value.multiplier"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(cat_name: string, multiplier: number, values: number[], sum: number) { return {category: cat_name, multiplier, computed_values: values, total: sum}; @@ -602,7 +602,7 @@ async fn test_flow_object_transformations(db: Pool) -> anyhow::Result< modules: vec![ flow_module("source", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -633,7 +633,7 @@ export function main() { js_input("active_bonus", "results.source.config.activeBonus"), js_input("all_users", "results.source.users"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(admins: string[], active_count: number, admin_bonus: number, active_bonus: number, all_users: any[]) { // Compute user_summary and total_bonus in the script since complex expressions in input_transforms @@ -702,7 +702,7 @@ async fn test_flow_skip_if_expressions(db: Pool) -> anyhow::Result<()> "check", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {should_skip: true, value: 100}; @@ -722,7 +722,7 @@ export function main() { "maybe_skipped", FlowModuleValue::RawScript { input_transforms: [js_input("input_val", "results.check.value * 2")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(input_val: number) { return {processed: input_val, was_run: true}; @@ -750,7 +750,7 @@ export function main(input_val: number) { js_input("prev", "previous_result"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(check_val: number, prev: any) { return {check_val, previous: prev}; @@ -799,7 +799,7 @@ async fn test_flow_template_literals(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -825,7 +825,7 @@ export function main() { js_input("upper_name", "results.data.firstName.toUpperCase()"), js_input("items_formatted", "`Items: ${results.data.items.map(i => i.charAt(0).toUpperCase() + i.slice(1)).join(', ')}`"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(full_name: string, greeting: string, items_str: string, upper_name: string, items_formatted: string) { return {full_name, greeting, items_str, upper_name, items_formatted}; @@ -873,7 +873,7 @@ async fn test_flow_optional_chaining(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -905,7 +905,7 @@ export function main() { js_input("default_city", "results.data.partial_user?.address?.city ?? 'Unknown'"), js_input("default_country", "results.data.user?.address?.country ?? 'USA'"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(city: string, missing_city: any, null_user_name: any, default_city: string, default_country: string) { return {city, missing_city, null_user_name, default_city, default_country}; @@ -955,7 +955,7 @@ async fn test_flow_parallel_forloop(db: Pool) -> anyhow::Result<()> { "data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {items: [1, 2, 3, 4, 5]}; @@ -990,7 +990,7 @@ export function main() { js_input("cubed", "flow_input.iter.value.value ** 3"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(original: number, squared: number, cubed: number) { return {original, squared, cubed}; @@ -1089,7 +1089,7 @@ async fn test_flow_env_access(db: Pool) -> anyhow::Result<()> { js_input("retry_count", "flow_env.CONFIG.retries"), js_input("has_auth", "flow_env.CONFIG.features.includes('auth')"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(env_name: string, is_debug: boolean, timeout_val: number, api_url: string, retry_count: number, has_auth: boolean) { return {env_name, is_debug, timeout_val, api_url, retry_count, has_auth}; @@ -1158,7 +1158,7 @@ async fn test_flow_input_and_env_combined(db: Pool) -> anyhow::Result< // Conditional based on both js_input("multiplier", "flow_env.ENV === 'production' ? flow_input.prod_mult : 1"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(effective_limit: number, env_prefix: string, is_prod: boolean, doubled_input: number, multiplier: number) { return {effective_limit, env_prefix, is_prod, doubled_input, final_value: doubled_input * multiplier}; @@ -1215,7 +1215,7 @@ async fn test_flow_results_optional_chaining(db: Pool) -> anyhow::Resu "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1287,7 +1287,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( user_name: string, @@ -1361,7 +1361,7 @@ async fn test_flow_large_integers(db: Pool) -> anyhow::Result<()> { "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1401,7 +1401,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( small: number, i32_max: number, over_i32: number, @@ -1459,7 +1459,7 @@ async fn test_flow_unicode_emoji(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1489,7 +1489,7 @@ export function main() { js_input("mixed_upper", "results.a.mixed.toUpperCase()"), js_input("template", "`Welcome: ${results.a.greeting}`"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( greeting: string, greeting_len: number, simple_str: string, has_world: boolean, @@ -1544,7 +1544,7 @@ async fn test_flow_complex_array_operations(db: Pool) -> anyhow::Resul modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1585,7 +1585,7 @@ export function main() { js_input("first_three", "results.a.numbers.slice(0, 3)"), js_input("last_two", "results.a.numbers.slice(-2)"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( sorted_asc: number[], sorted_desc: number[], active_names: string[], @@ -1651,7 +1651,7 @@ async fn test_flow_multiline_expressions(db: Pool) -> anyhow::Result<( modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1710,7 +1710,7 @@ export function main() { } "#), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( subtotal: number, discounted_total: number, @@ -1768,7 +1768,7 @@ async fn test_flow_spread_with_results(db: Pool) -> anyhow::Result<()> modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1803,7 +1803,7 @@ export function main() { // Combining multiple spreads js_input("combined", "{config: {...results.a.config}, tags: [...results.a.tags], source: 'flow'}"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( merged_config: any, all_tags: string[], full_user: any, @@ -1869,7 +1869,7 @@ async fn test_flow_nested_forloop_results_access(db: Pool) -> anyhow:: "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1908,7 +1908,7 @@ export function main() { js_input("multiplier", "results.a.multiplier"), // Access outer step from inside for-loop ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(category: string, multiplier: number) { return { @@ -1939,7 +1939,7 @@ export function main(category: string, multiplier: number) { js_input("original_mult", "results.a.multiplier"), // Access outer step ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(items: any[], category: string, original_mult: number) { return { @@ -2013,7 +2013,7 @@ async fn test_flow_results_non_existent_step(db: Pool) -> anyhow::Resu "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { value: 42 }; @@ -2050,7 +2050,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( existing: number, @@ -2126,7 +2126,7 @@ async fn test_flow_env_skip_if_inside_branchone(db: Pool) -> anyhow::R "inner", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2156,7 +2156,7 @@ export function main() { "branch_marker", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {marker: "branch-marker"}; @@ -2193,7 +2193,7 @@ export function main() { "after", FlowModuleValue::RawScript { input_transforms: [js_input("prev", "previous_result")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev: any) { return {prev}; @@ -2258,7 +2258,7 @@ async fn test_flow_env_skip_if_nested_branchone(db: Pool) -> anyhow::R "leaf", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2282,7 +2282,7 @@ export function main() { "inner_marker", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {marker: "inner"}; @@ -2337,7 +2337,7 @@ export function main() { "after", FlowModuleValue::RawScript { input_transforms: [js_input("prev", "previous_result")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev: any) { return {prev}; @@ -2414,7 +2414,7 @@ async fn test_flow_env_complex_input_transform_in_branch(db: Pool) -> "Math.min(flow_env.LIMIT, 10) + flow_env.OFFSET", )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(value: number) { return {value}; @@ -2487,7 +2487,7 @@ async fn test_flow_env_skip_if_in_parallel_forloop(db: Pool) -> anyhow "marker", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return {marker: i}; @@ -2508,7 +2508,7 @@ export function main(i: number) { "leaf", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2594,7 +2594,7 @@ async fn test_flow_env_imported_flow_uses_own_env(db: Pool) -> anyhow: "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(key: string) { return {key}; }", "input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } @@ -2679,7 +2679,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool) -> "id": "marker", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {marker: \"from-imported\"}; }", "input_transforms": {} } @@ -2688,7 +2688,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool) -> "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {ran: true}; }", "input_transforms": {} }, @@ -2791,7 +2791,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> a "id": "marker", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {marker: \"from-imported-branch\"}; }", "input_transforms": {} } @@ -2800,7 +2800,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> a "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {ran: true}; }", "input_transforms": {} }, @@ -2932,13 +2932,13 @@ async fn test_flow_env_marks_sub_flows_only_without_ancestor_env( "parallel": true, "modules": [{ "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(key: any) { return key ?? null; }", "input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } } }}, { "id": "tail", "value": { // A single simple step is inlined rather than stored as a flow node. "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(prev: any) { return prev; }", "input_transforms": { "prev": { "type": "javascript", "expr": "results.leaf" } } }}] @@ -3024,7 +3024,7 @@ async fn test_flow_env_in_stop_after_if(db: Pool) -> anyhow::Result<() "first", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "first"}; @@ -3052,7 +3052,7 @@ export function main() { "second", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "second"}; @@ -3110,7 +3110,7 @@ async fn test_stop_after_if_error_include_result(db: Pool) -> anyhow:: "step", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { userErrors: ["email taken"], ok: false }; @@ -3200,7 +3200,7 @@ async fn test_flow_env_in_retry_if(db: Pool) -> anyhow::Result<()> { "fails", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { throw new Error("nope"); @@ -3274,7 +3274,7 @@ async fn test_flow_env_in_stop_after_all_iters_if(db: Pool) -> anyhow: "iter_step", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return {iter: i}; @@ -3316,7 +3316,7 @@ export function main(i: number) { "after", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "after-loop"}; @@ -3375,7 +3375,7 @@ async fn test_stop_after_all_iters_if_error_includes_result( "iter_step", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return { iter: i }; @@ -3453,8 +3453,8 @@ async fn test_flow_expr_step_tag(db: Pool) -> anyhow::Result<()> { id, FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, - content: "export function main() { return { lang: 'bun' } }".to_string(), + language: ScriptLang::Bun, + content: "export function main() { return { lang: 'python3' } }".to_string(), path: None, lock: None, tag: tag.map(str::to_string), @@ -3484,7 +3484,7 @@ async fn test_flow_expr_step_tag(db: Pool) -> anyhow::Result<()> { .bind(job.id) .fetch_one(&db) .await?; - assert_eq!(b_tag, "bun"); + assert_eq!(b_tag, "python3"); assert!(!job.success); let result = job.json_result().unwrap(); diff --git a/backend/tests/instance_config.rs b/backend/tests/instance_config.rs index 52ebbee7e7..24937a3ce5 100644 --- a/backend/tests/instance_config.rs +++ b/backend/tests/instance_config.rs @@ -237,7 +237,7 @@ async fn test_diff_global_settings_empty_string_routes_to_delete(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "key_a").await, @@ -468,7 +468,7 @@ async fn test_apply_settings_diff_deletes_only(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "to_delete_1").await.is_none()); assert!(get_global_setting(&db, "to_delete_2").await.is_none()); @@ -489,7 +489,7 @@ async fn test_apply_settings_diff_upserts_and_deletes(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "old_key").await.is_none()); assert_eq!( @@ -503,7 +503,7 @@ async fn test_apply_settings_diff_empty_noop(db: Pool) { insert_global_setting(&db, "preexisting", serde_json::json!("value")).await; let diff = SettingsDiff::default(); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "preexisting").await, @@ -525,7 +525,7 @@ async fn test_apply_settings_diff_upsert_overwrites(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "overwrite_me").await, @@ -553,7 +553,7 @@ async fn test_apply_settings_diff_complex_json(db: Pool) { ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); let stored = get_global_setting(&db, "complex_setting").await.unwrap(); assert_eq!(stored, complex_value); @@ -568,7 +568,7 @@ async fn test_apply_settings_diff_delete_nonexistent_is_noop(db: Pool) }; // Should not error - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); } // ======================================================================== @@ -589,7 +589,7 @@ async fn test_apply_configs_diff_upserts_with_prefix(db: Pool) { deletes: vec![], }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); let stored = get_config(&db, "worker__mygroup").await.unwrap(); assert_eq!(stored["init_bash"], "echo hi"); @@ -601,7 +601,7 @@ async fn test_apply_configs_diff_deletes_with_prefix(db: Pool) { let diff = ConfigsDiff { upserts: BTreeMap::new(), deletes: vec!["to_remove".to_string()] }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); assert!(get_config(&db, "worker__to_remove").await.is_none()); } @@ -619,7 +619,7 @@ async fn test_apply_configs_diff_upsert_overwrites(db: Pool) { deletes: vec![], }; - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); let stored = get_config(&db, "worker__grp").await.unwrap(); assert_eq!(stored, serde_json::json!({"new": true})); @@ -630,7 +630,7 @@ async fn test_apply_configs_diff_empty_noop(db: Pool) { insert_config(&db, "worker__keep", serde_json::json!({"keep": true})).await; let diff = ConfigsDiff::default(); - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); assert!(get_config(&db, "worker__keep").await.is_some()); } @@ -641,7 +641,7 @@ async fn test_apply_configs_diff_does_not_touch_non_worker_configs(db: Pool) { assert!(diff.upserts.contains_key("expose_metrics")); assert!(diff.deletes.is_empty()); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Step 5: Read back and verify let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -729,7 +729,7 @@ async fn test_roundtrip_replace_mode_deletes(db: Pool) { assert!(diff.deletes.contains(&"retention_period_secs".to_string())); assert!(diff.deletes.contains(&"expose_metrics".to_string())); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert!(get_global_setting(&db, "retention_period_secs") .await @@ -782,7 +782,7 @@ async fn test_roundtrip_worker_configs(db: Pool) { assert!(diff.upserts.contains_key("gpu")); // new assert_eq!(diff.deletes, vec!["legacy".to_string()]); - apply_configs_diff(&db, &diff).await.unwrap(); + apply_configs_diff(&db, &diff, &()).await.unwrap(); // Verify let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -820,7 +820,7 @@ async fn test_roundtrip_to_settings_map_from_db_consistency(db: Pool) let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); let config = InstanceConfig::from_db(&db).await.unwrap(); assert_eq!(config.global_settings.base_url, original.base_url); @@ -868,8 +868,8 @@ async fn test_idempotent_apply(db: Pool) { }; // Apply twice - apply_settings_diff(&db, &diff).await.unwrap(); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); assert_eq!( get_global_setting(&db, "idem_key").await, @@ -1069,7 +1069,7 @@ async fn test_diff_apply_only_touches_changed_rows(db: Pool) { assert!(!diff.upserts.contains_key("unchanged_1")); assert!(!diff.upserts.contains_key("unchanged_2")); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Verify all 4 settings present assert_eq!(count_global_settings(&db).await, 4); @@ -1148,7 +1148,7 @@ async fn test_replace_mode_protects_settings_in_integration(db: Pool) // But normal_setting should be deleted assert!(diff.deletes.contains(&"normal_setting".to_string())); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Verify protected settings survived assert!(get_global_setting(&db, "ducklake_settings").await.is_some()); @@ -1196,7 +1196,7 @@ async fn test_jwt_secret_roundtrip_as_string_or_secret_ref(db: Pool) { let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Read back from DB — jwt_secret should survive the roundtrip let config = InstanceConfig::from_db(&db).await.unwrap(); @@ -1239,7 +1239,7 @@ async fn test_rsa_keys_roundtrip_via_extra(db: Pool) { let map = settings.to_settings_map(); let diff = SettingsDiff { upserts: map.into_iter().collect(), deletes: vec![], ..Default::default() }; - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Read back from DB let config = InstanceConfig::from_db(&db).await.unwrap(); @@ -1296,7 +1296,7 @@ async fn test_replace_mode_protects_jwt_secret_and_rsa_keys(db: Pool) "normal_setting should be deleted" ); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Both protected settings survive assert_eq!( @@ -1371,7 +1371,7 @@ async fn test_alert_config_in_global_settings_roundtrip(db: Pool) { "alert_config change should be detected in diff" ); - apply_settings_diff(&db, &diff).await.unwrap(); + apply_settings_diff(&db, &diff, &()).await.unwrap(); // Re-read and verify the update let config2 = InstanceConfig::from_db(&db).await.unwrap(); @@ -1455,6 +1455,7 @@ async fn declarative_sync_rejects_an_unusable_webhook_base_url(db: Pool &db, &BTreeMap::new(), &desired, + &(), ) .await .expect_err("a javascript: banner link must fail the sync"); @@ -1545,6 +1548,7 @@ async fn declarative_sync_rejects_a_malformed_max_token_expiration(db: Pool) { &db, &BTreeMap::new(), &desired, + &(), ) .await .expect_err("a non-hex accent color must fail the sync"); @@ -1681,7 +1691,109 @@ async fn declarative_sync_rejects_a_non_hex_accent_color(db: Pool) { &db, &BTreeMap::new(), &desired, + &(), ) .await .expect("a hex accent color must sync"); } + +/// A declarative sync has no user request behind it, so its observer is the only thing +/// that audits it: one entry per setting or worker config it actually writes, values +/// recorded under the API path's rule, and none for what it leaves as is. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_declarative_sync_audits_only_what_it_writes(db: Pool) { + use windmill_audit::instance_config::{ + declarative_sync_author, DeclarativeSource, InstanceConfigAuditor, + }; + + insert_global_setting(&db, "retention_period_secs", serde_json::json!(100)).await; + insert_global_setting(&db, "expose_metrics", serde_json::json!(true)).await; + insert_global_setting(&db, "scim_token", serde_json::json!("old-scim-secret")).await; + insert_global_setting( + &db, + "npm_config_registry", + serde_json::json!("https://u:p@npm"), + ) + .await; + insert_config( + &db, + "worker__kept", + serde_json::json!({"worker_tags": ["a"]}), + ) + .await; + insert_config( + &db, + "worker__dropped", + serde_json::json!({"worker_tags": ["b"]}), + ) + .await; + + let config: windmill_common::instance_config::InstanceConfig = + serde_json::from_value(serde_json::json!({ + "global_settings": { + "retention_period_secs": 200, + "expose_metrics": true, + "scim_token": "new-scim-secret", + }, + "worker_configs": { + "kept": {"worker_tags": ["a"]}, + "added": {"init_bash": "export TOKEN=init-secret"}, + }, + })) + .unwrap(); + let author = declarative_sync_author(); + config + .sync_to_db( + &db, + &InstanceConfigAuditor { + db: &db, + author: &author, + source: Some(DeclarativeSource::SyncConfig), + }, + ) + .await + .unwrap(); + + let entries: Vec<(String, String, serde_json::Value)> = sqlx::query_as( + "SELECT operation, resource, parameters FROM audit_partitioned \ + WHERE workspace_id = 'global' AND username = $1 AND resource = ANY($2) \ + ORDER BY resource", + ) + .bind(&author.username) + .bind(vec![ + "retention_period_secs", + "expose_metrics", + "scim_token", + "npm_config_registry", + "worker__kept", + "worker__dropped", + "worker__added", + ]) + .fetch_all(&db) + .await + .unwrap(); + + let source = serde_json::json!({"source": "sync-config"}); + let entry = |operation: &str, resource: &str, parameters: &serde_json::Value| { + ( + operation.to_string(), + resource.to_string(), + parameters.clone(), + ) + }; + assert_eq!( + entries, + vec![ + entry("global_settings.delete", "npm_config_registry", &source), + entry( + "global_settings.update", + "retention_period_secs", + &serde_json::json!({"source": "sync-config", "value": "200"}), + ), + entry("global_settings.update", "scim_token", &source), + entry("worker_config.update", "worker__added", &source), + entry("worker_config.delete", "worker__dropped", &source), + ] + ); +} diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index 0038effc78..c40feb1650 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -54,7 +54,7 @@ mod job_payload { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, job_token_scopes: None, @@ -85,7 +85,7 @@ mod job_payload { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: true, concurrency_settings: @@ -165,7 +165,7 @@ mod job_payload { let test = || async { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[0], - language: ScriptLang::Deno, + language: ScriptLang::Bun, concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), cache_ttl: None, @@ -185,7 +185,7 @@ mod job_payload { let test = || async { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[1], - language: ScriptLang::Deno, + language: ScriptLang::Bun, concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), cache_ttl: None, @@ -263,7 +263,7 @@ mod job_payload { let result = RunJob::from(JobPayload::Dependencies { path: "f/system/hello".to_string(), hash: ScriptHash(123412), - language: ScriptLang::Deno, + language: ScriptLang::Bun, debouncing_settings: Default::default(), dedicated_worker: None, }) @@ -351,7 +351,7 @@ mod job_payload { console.log(greet) return greet }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -407,7 +407,7 @@ mod job_payload { return greet }"# .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, }) .run_until_complete(&db, false, port) .await @@ -416,7 +416,10 @@ mod job_payload { assert_eq!( result, - json!({ "lock": "", "status": "Successful lock file generation" }) + json!({ + "lock": "{\n \"dependencies\": {}\n}\n//bun.lock\n", + "status": "Successful lock file generation" + }) ); }; test_for_versions(VERSION_FLAGS.iter().copied(), test).await; @@ -626,7 +629,7 @@ mod job_payload { console.log(greet) return greet }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -673,7 +676,7 @@ mod job_payload { "content": r#"export function main(world: string) { return `Hello ${world}!`; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -685,7 +688,7 @@ mod job_payload { "content": r#"export function main(world: string, a: string) { return `${a} ${world}!`; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" }, "a": { "type": "javascript", "expr": "results.a" } @@ -704,7 +707,7 @@ mod job_payload { "x": { "type": "javascript", "expr": "flow_input.iter.value" } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": r#"export function main(world: string, b: string, x: string) { return `${x}: ${b} ${world}!`; }"#, @@ -857,7 +860,7 @@ mod job_payload { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } }, @@ -874,7 +877,7 @@ mod job_payload { "id": "inner_first", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } }, @@ -884,7 +887,7 @@ mod job_payload { "id": "inner_second", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" }, "first": { "type": "javascript", "expr": "results.inner_first" } @@ -1045,7 +1048,7 @@ mod job_payload { "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "iter_val": { "type": "javascript", "expr": "flow_input.iter.value" }, "tag": { "type": "javascript", "expr": "flow_input.tag" } @@ -1244,7 +1247,7 @@ mod job_payload { "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "tag": { "type": "javascript", "expr": "flow_input.tag" }, "iv": { "type": "javascript", "expr": "flow_input.iter.value" } @@ -1388,7 +1391,7 @@ mod job_payload { "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "tag": { "type": "javascript", "expr": "flow_input.tag" } }, diff --git a/backend/tests/job_token_scopes.rs b/backend/tests/job_token_scopes.rs index 6af694bff0..f805168bfa 100644 --- a/backend/tests/job_token_scopes.rs +++ b/backend/tests/job_token_scopes.rs @@ -410,5 +410,22 @@ async fn test_oidc_claim_carries_the_presented_scopes(db: Pool) -> any )?; assert_eq!(claims["job_token_scopes"], expected, "{job}"); } + + // A soft-canceled job is still `running` until its worker kills it. + sqlx::query("UPDATE v2_job_queue SET canceled_by = 'test-user' WHERE id = $1") + .bind(Uuid::parse_str(OIDC_JOB)?) + .execute(&db) + .await?; + let resp = client + .post(format!("{base}/oidc/token/test-audience")) + .bearer_auth(job_token(&db, OIDC_JOB).await?) + .send() + .await?; + assert_eq!( + resp.status(), + StatusCode::FORBIDDEN, + "{}", + resp.text().await? + ); Ok(()) } diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs index 6a101200a1..bffacfbaf0 100644 --- a/backend/tests/jobs_read_auth.rs +++ b/backend/tests/jobs_read_auth.rs @@ -476,6 +476,44 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul ); } + // A lock job belongs to `jobs:run:dependencies`: its path is whatever the caller sent, + // so a token scoped to run the flow at that path must not read it. + let path = "completed/get_result/19191919-1919-1919-1919-191919191919"; + let (status, body) = get(&base, path, Some("RUN_SCOPED_DEPS_TOKEN")).await; + assert!( + status.is_success() && body.contains("DEPS_RESULT"), + "dependencies-scoped token must read its lock job (got {status}): {body}" + ); + let (status, body) = get(&base, path, Some("RUN_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a flow-scoped token must not read a lock job (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("completed/get_result/{FLOW_JOB}"), + Some("RUN_SCOPED_DEPS_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "dependencies-scoped token must not read a flow run at its path (got {status}): {body}" + ); + // A lock job filed at `*` matches no path-scoped grant: the stored path is not a pattern. + let (status, body) = get( + &base, + "completed/get_result/20202020-2020-2020-2020-202020202020", + Some("RUN_SCOPED_DEPS_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a path-scoped token must not read a lock job filed at `*` (got {status}): {body}" + ); + // An `apps:run:` scope is a start grant too: the inline-script component run it // launched — a kind no `jobs:run` scope can name — stays readable to a token scoped // to that app, and stays out of reach for one that is only scoped to run jobs. diff --git a/backend/tests/mcp_header_var_refs.rs b/backend/tests/mcp_header_var_refs.rs new file mode 100644 index 0000000000..158bc71a5f --- /dev/null +++ b/backend/tests/mcp_header_var_refs.rs @@ -0,0 +1,110 @@ +//! An MCP resource header whose value is a `$var:` reference reaches the MCP +//! server as the variable's value, and only for a caller allowed to read it. +#![cfg(feature = "mcp")] + +use sqlx::{Pool, Postgres}; +use tokio::io::AsyncReadExt; +use windmill_common::client::AuthedClient; +use windmill_test_utils::*; +use windmill_worker::{load_mcp_tools, McpResourceConfig}; + +const SECRET_VALUE: &str = "S3CRET-MCP-HEADER-VALUE"; + +/// A secret variable and an MCP resource at `url` sending it as a header. +async fn seed(db: &Pool, url: &str) -> anyhow::Result<()> { + let mc = windmill_common::variables::build_crypt(db, "test-workspace").await?; + sqlx::query( + "INSERT INTO variable (workspace_id, path, value, is_secret, description, extra_perms) + VALUES ('test-workspace', 'u/test-user/mcp_header', $1, true, '', '{}')", + ) + .bind(windmill_common::variables::encrypt(&mc, SECRET_VALUE)) + .execute(db) + .await?; + sqlx::query( + "INSERT INTO resource (workspace_id, path, value, resource_type, extra_perms, created_by) + VALUES ('test-workspace', 'u/test-user/mcp', $1, 'mcp', '{}', 'test-user')", + ) + .bind(serde_json::json!({ + "name": "mcp", + "url": url, + "headers": { "X-Api-Key": "$var:u/test-user/mcp_header", "X-Plain": "plain" } + })) + .execute(db) + .await?; + Ok(()) +} + +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_is_resolved(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + std::env::set_var("ALLOW_PRIVATE_MCP_SERVER_URLS", "true"); + + // Not an MCP server: it captures the first request and hangs up, so loading + // fails at the connect step after the headers were sent. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let mcp_port = listener.local_addr()?.port(); + let first_request = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await?; + let mut buf = vec![0u8; 16 * 1024]; + let n = socket.read(&mut buf).await?; + anyhow::Ok(String::from_utf8_lossy(&buf[..n]).to_lowercase()) + }); + seed(&db, &format!("http://127.0.0.1:{mcp_port}/mcp")).await?; + + let server = ApiServer::start(db.clone()).await?; + let client = AuthedClient::new( + format!("http://localhost:{}", server.addr.port()), + "test-workspace".to_string(), + "SECRET_TOKEN".to_string(), + None, + ); + let config = vec![McpResourceConfig { + resource_path: "$res:u/test-user/mcp".to_string(), + include_tools: None, + exclude_tools: None, + }]; + let _ = load_mcp_tools(&db, "test-workspace", config, &client).await; + + let request = first_request.await??; + assert!( + request.contains(&format!("x-api-key: {}", SECRET_VALUE.to_lowercase())), + "the header must carry the variable's value, not the reference: {request}" + ); + assert!(request.contains("x-plain: plain"), "{request}"); + + Ok(()) +} + +/// A token scoped to the resource alone must not have the API read the +/// variable a header points at. +#[sqlx::test(fixtures("base"))] +async fn test_mcp_header_var_ref_requires_variable_scope(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + seed(&db, "https://mcp.invalid.windmill.test").await?; + sqlx::query( + "INSERT INTO token (token_hash, token_prefix, token, email, label, super_admin, scopes) + VALUES (encode(sha256('RES_ONLY_TOKEN'::bytea), 'hex'), 'RES_ONLY_T', 'RES_ONLY_TOKEN', + 'test@windmill.dev', 'resources only', false, '{resources:read}')", + ) + .execute(&db) + .await?; + + let server = ApiServer::start(db.clone()).await?; + let resp = reqwest::Client::new() + .get(format!( + "http://localhost:{}/api/w/test-workspace/resources/mcp_tools/u/test-user/mcp", + server.addr.port() + )) + .header("Authorization", "Bearer RES_ONLY_TOKEN") + .send() + .await?; + let status = resp.status(); + let body = resp.text().await?; + assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "{body}"); + assert!( + body.contains("variables:read:u/test-user/mcp_header"), + "{body}" + ); + + Ok(()) +} diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index f1e042037f..30585c08a1 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -176,6 +176,10 @@ async fn test_all_metrics_e2e() { otel_set_worker_uptime("w-uptime", 3600.0); otel_set_health_status_phase("healthy"); otel_set_health_db_unresponsive(true); + otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024)); + otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35); + otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25); + otel_record_job_memory_peak("python3", 100 * 1024 * 1024); // ── Histograms ────────────────────────────────────────────────── @@ -189,7 +193,7 @@ async fn test_all_metrics_e2e() { let metrics = flush_and_get_metrics(&state); let names = metric_names(&metrics); - // ── Verify all 21 metric names are present ────────────────────── + // ── Verify all metric names are present ───────────────────────── let expected = [ "windmill.queue.push_count", @@ -213,6 +217,15 @@ async fn test_all_metrics_e2e() { "windmill.worker.uptime", "windmill.health.status", "windmill.health.db_unresponsive", + "windmill.worker.memory_usage", + "windmill.worker.windmill_memory_usage", + "windmill.worker.memory_limit", + "windmill.worker.cpu_usage", + "windmill.worker.cpu_periods", + "windmill.worker.cpu_throttled_periods", + "windmill.worker.cpu_throttled_time", + "windmill.worker.occupancy_rate", + "windmill.worker.job_memory_peak", ]; for name in expected { assert!( @@ -290,6 +303,18 @@ async fn test_all_metrics_e2e() { .expect("worker.busy data point with worker=worker-test-1 not found"); assert_eq!(dp.1, 1); + let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu") + }) + .expect("worker.memory_limit data point with worker_group=gpu not found"); + assert_eq!(dp.1, 1024); + let m = find_metric(&metrics, "windmill.db.pool.active").unwrap(); assert_eq!(gauge_i64_values(m)[0].1, 5); let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap(); diff --git a/backend/tests/restart_at_step_api.rs b/backend/tests/restart_at_step_api.rs index 2b57de2c64..f6fc2a04d3 100644 --- a/backend/tests/restart_at_step_api.rs +++ b/backend/tests/restart_at_step_api.rs @@ -217,7 +217,7 @@ async fn test_api_restart_at_step_rejects_parallel_loop(db: Pool) -> a "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { return 'ok' }" } diff --git a/backend/tests/retry.rs b/backend/tests/retry.rs index e804b96f0e..d80e386b61 100644 --- a/backend/tests/retry.rs +++ b/backend/tests/retry.rs @@ -70,12 +70,20 @@ mod retry { fn inner_step() -> &'static str { r#" +import { connect } from "node:net"; + export async function main(index, port) { - const buf = new Uint8Array([0]); - const sock = await Deno.connect({ port }); - await sock.write(new Uint8Array([index])); - if (await sock.read(buf) != 1) throw Error("read"); - return buf[0]; + return await new Promise((resolve, reject) => { + const sock = connect({ port, host: "127.0.0.1" }, () => { + sock.write(new Uint8Array([index])); + }); + sock.on("data", (data) => { + resolve(data[0]); + sock.destroy(); + }); + sock.on("error", reject); + sock.on("close", () => reject(Error("read"))); + }); } "# } @@ -105,7 +113,7 @@ def main(last, port): "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": inner_step(), }, }], @@ -281,7 +289,7 @@ def main(last, port): "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return \"ok\" }", }, }, { @@ -293,7 +301,7 @@ def main(last, port): "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": inner_step(), }, "retry": { "constant": { "attempts": 1, "seconds": 0 } }, diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs index 8fc4182d9e..14df1a4c25 100644 --- a/backend/tests/success_handler.rs +++ b/backend/tests/success_handler.rs @@ -2,6 +2,36 @@ use sqlx::{Pool, Postgres}; use windmill_test_utils::*; +/// Both job-running tests seed `f/test/success_handler` with a different hash, each in its own +/// database, while the deployed-script caches are process-global and keyed by workspace and +/// path. With them on, one test resolves the handler to the other's hash and the handler push +/// 404s whenever that hash's info is not cached yet. +#[cfg(all(feature = "enterprise", feature = "private"))] +fn disable_deployed_script_cache() { + windmill_common::DEPLOYED_SCRIPT_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); +} + +/// The handler job is pushed after the job it handles is reported complete. +#[cfg(all(feature = "enterprise", feature = "private"))] +async fn wait_for_success_handler_job( + db: &Pool, +) -> anyhow::Result, Option)>> { + for _ in 0..100 { + let job = sqlx::query_as::<_, (Option, Option)>( + "SELECT runnable_path, parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'", + ) + .fetch_optional(db) + .await?; + if job.is_some() { + return Ok(job); + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + Ok(None) +} + /// Test that the workspace success handler cache works correctly with 60s TTL #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] @@ -108,6 +138,7 @@ async fn test_success_handler_triggered_on_success(db: Pool) -> anyhow use windmill_common::scripts::{ScriptHash, ScriptLang}; initialize_tracing().await; + disable_deployed_script_cache(); let server = ApiServer::start(db.clone()).await?; @@ -123,7 +154,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '') + VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\n//bun.lock\n') "#, success_handler_code ) @@ -138,7 +169,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '') + VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\n//bun.lock\n') "#, test_script_hash, test_script_code @@ -175,7 +206,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -191,51 +222,72 @@ export async function main(path: string, email: string, job_id: string, is_flow: let main_job_id = completed_job.id; - // Wait a short time for the success handler job to be created - tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; - - // Verify the success handler job was created (query by email since trigger is not set) - let success_handler_job = sqlx::query!( - r#" - SELECT - id, - runnable_path, - permissioned_as_email, - parent_job - FROM v2_job - WHERE workspace_id = 'test-workspace' - AND permissioned_as_email = 'success_handler@windmill.dev' - ORDER BY created_at DESC - LIMIT 1 - "# - ) - .fetch_optional(&db) - .await?; - - assert!( - success_handler_job.is_some(), - "Success handler job should have been created" - ); - - let handler_job = success_handler_job.unwrap(); - - // Verify the success handler job has correct parameters + let handler_job = wait_for_success_handler_job(&db).await?; assert_eq!( - handler_job.runnable_path.as_deref(), - Some("f/test/success_handler"), - "Success handler should run the configured script" - ); - assert_eq!( - handler_job.permissioned_as_email.as_str(), - "success_handler@windmill.dev", - "Success handler should run as success_handler user" - ); - assert_eq!( - handler_job.parent_job, - Some(main_job_id), - "Success handler should have main job as parent" + handler_job, + Some(( + Some("f/test/success_handler".to_string()), + Some(main_job_id) + )), + "Success handler should run the configured script with the main job as parent" ); // Note: root_job may be None when it equals parent_job (optimization in push function) Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its success must reach the workspace success handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_triggered_on_hub_script_success( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + disable_deployed_script_cache(); + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000003"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/ok"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(completed_job.success, "Hub script should have succeeded"); + + let handler_job = wait_for_success_handler_job(&db).await?; + assert_eq!( + handler_job.map(|(_, parent)| parent), + Some(Some(completed_job.id)), + "Workspace success handler should handle the successful hub script" + ); + + Ok(()) +} diff --git a/backend/tests/suspend_resume.rs b/backend/tests/suspend_resume.rs index 6fb5d68004..0decc16519 100644 --- a/backend/tests/suspend_resume.rs +++ b/backend/tests/suspend_resume.rs @@ -63,11 +63,11 @@ mod suspend_resume { "op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "\ export async function main(n, port, op) {\ - const job = Deno.env.get('WM_JOB_ID'); - const token = Deno.env.get('WM_TOKEN'); + const job = process.env.WM_JOB_ID; + const token = process.env.WM_TOKEN; const r = await fetch( `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ {\ @@ -102,7 +102,7 @@ mod suspend_resume { "resumes": { "type": "javascript", "expr": "resumes", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }" }, "suspend": { @@ -116,7 +116,7 @@ mod suspend_resume { "resumes": { "type": "javascript", "expr": "resumes", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(last, resume, resumes) { return { last, resume, resumes } }" }, }], @@ -256,11 +256,11 @@ mod suspend_resume { "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "\ export async function main(port) {\ - const job = Deno.env.get('WM_JOB_ID');\ - const token = Deno.env.get('WM_TOKEN');\ + const job = process.env.WM_JOB_ID;\ + const token = process.env.WM_TOKEN;\ const secret = await (await fetch(\ `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ { headers: { 'Authorization': `Bearer ${token}` } }\ @@ -278,7 +278,7 @@ mod suspend_resume { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'b ran' }", }, // The gate holds `b` back, so `b` never runs and its error policy describes @@ -290,7 +290,7 @@ mod suspend_resume { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'handled' }", }, }, @@ -343,7 +343,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -355,7 +355,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -454,7 +454,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -466,7 +466,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -556,7 +556,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -567,7 +567,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -658,7 +658,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -670,7 +670,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -764,7 +764,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -776,7 +776,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -933,7 +933,7 @@ mod suspend_resume { "suspend": { "required_events": 1, "timeout": 86400 }, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 1 }", "input_transforms": {}, }, @@ -942,7 +942,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 42 }", "input_transforms": {}, }, @@ -980,7 +980,7 @@ mod suspend_resume { "suspend": { "required_events": APPROVALS, "timeout": 86400 }, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 1 }", "input_transforms": {}, }, @@ -989,7 +989,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(resumes) { return resumes.length }", "input_transforms": { "resumes": { "type": "javascript", "expr": "resumes" }, diff --git a/backend/tests/wac_suspend_started_at.rs b/backend/tests/wac_suspend_started_at.rs index 5a83730d24..ca364e5993 100644 --- a/backend/tests/wac_suspend_started_at.rs +++ b/backend/tests/wac_suspend_started_at.rs @@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool) -> anyhow::Res Ok(()) } + +/// Each park clears `started_at`, so the completion would otherwise store only the last +/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that +/// started an hour late. The completed row spans the workflow from its first start, and +/// keeps the last segment aside: that is what is metered and what worker-time readers sum. +#[sqlx::test] +async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment( + db: Pool, +) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \ + VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \ + 'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)", + ) + .bind(job_id) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \ + VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')", + ) + .bind(job_id) + .execute(&db) + .await?; + + // Two parks, each followed by the pull re-stamping `started_at`: the second park must + // not replace the first start with its own segment's. + for resumed_ago in ["10 minutes", "3 seconds"] { + let mut tx = db.begin().await?; + suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?; + tx.commit().await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \ + WHERE id = $1", + ) + .bind(job_id) + .bind(resumed_ago) + .execute(&db) + .await?; + } + + let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db) + .await? + .unwrap(); + let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?; + let (_, span) = windmill_queue::add_completed_job( + &db, + &job, + true, + false, + sqlx::types::Json(&result), + None, + 0, + None, + false, + Some(3000), + false, + ) + .await?; + + let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option) = sqlx::query_as( + "SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \ + (extras->>'wac_last_segment_ms')::bigint \ + FROM v2_job_completed WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + assert!( + (started_ago_s - 3600.0).abs() < 60.0, + "started_at must be the first start, got {started_ago_s}s ago" + ); + assert!( + (duration_ms - 3_600_000).abs() < 60_000, + "duration_ms must be the wall time since the first start, got {duration_ms}" + ); + assert_eq!(last_segment_ms, Some(3000)); + assert_eq!( + span.duration_ms, duration_ms, + "a flow records the step with the span the row holds" + ); + assert!(span.outlasts_run(job.started_at)); + assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter"); + + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 23d88301b7..e75170ec6f 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -319,7 +319,7 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho ); } // ...and the `settings/global` keys on the allowlist, which the CLI reads from a job: on a - // git-sync push, and in `u/admin/hub_sync`. `ws_base_url` is the control: the handler + // git-sync push, and in `wmill hub pull`. `ws_base_url` is the control: the handler // leaves it as ungated as these, so only the allowlist stops it. for key in ["automate_username_creation", "uid", "hub_base_url"] { let resp = authed( diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 73d3fc2567..751752eb00 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -166,7 +166,7 @@ async fn test_iteration_parallel(db: Pool) -> anyhow::Result<()> { #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] -async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { +async fn test_bun_flow(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; @@ -183,7 +183,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { id: "a".to_string(), value: FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: numbers.to_string(), path: None, lock: None, @@ -232,7 +232,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { }, )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: doubles.to_string(), path: None, lock: None, @@ -296,7 +296,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { let port = server.addr.port(); for i in 0..50 { - println!("deno flow iteration: {}", i); + println!("bun flow iteration: {}", i); let job = run_job_in_new_worker_until_complete(&db, false, job.clone(), port).await; // println!("job: {:#?}", job.flow_status); let result = job.json_result().unwrap(); @@ -346,13 +346,13 @@ async fn test_identity(db: Pool) -> anyhow::Result<()> { #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] -async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { +async fn test_bun_flow_same_worker(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let write_file = r#"export async function main(loop: boolean, i: number, path: string) { - await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`); + await Bun.write(`./shared/${path}`, `${loop} ${i}`); }"# .to_string(); @@ -373,7 +373,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: write_file.clone(), path: None, lock: None, @@ -432,7 +432,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: write_file, path: None, lock: None, @@ -472,9 +472,9 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") }, )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#"export async function main(path: string, path2: string) { - return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`); + return await Bun.file(`./shared/${path}`).text() + "," + await Bun.file(`./shared/${path2}`).text(); }"# .to_string(), path: None, @@ -546,9 +546,9 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#"export async function main(path: string, loops: string[], path2: string) { - return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`); + return await Bun.file(`./shared/${path}`).text() + "," + loops + "," + await Bun.file(`./shared/${path2}`).text(); }"# .to_string(), path: None, @@ -618,7 +618,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool) -> anyhow::R "id": "c", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 1 }", } }] @@ -630,7 +630,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool) -> anyhow::R "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 42 }", } } @@ -671,7 +671,7 @@ async fn test_flow_result_by_id(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 42 }", } }, @@ -686,7 +686,7 @@ async fn test_flow_result_by_id(db: Pool) -> anyhow::Result<()> { "value": { "input_transforms": {"v": {"type": "javascript", "expr": "results.a"}}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(v){ return v }", } @@ -3857,7 +3857,7 @@ fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value { } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(array, i){ array.push(i); return array }", } }) @@ -3868,7 +3868,7 @@ fn module_failure() -> serde_json::Value { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ throw Error('failure') }", } }) @@ -3886,7 +3886,7 @@ async fn test_branchone_simple(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3924,7 +3924,7 @@ async fn test_branchone_with_cond(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3962,7 +3962,7 @@ async fn test_branchall_sequential(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4002,7 +4002,7 @@ async fn test_branchall_simple(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4050,7 +4050,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4086,7 +4086,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4132,7 +4132,7 @@ async fn test_branchone_nested(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [] }", } }, @@ -4191,7 +4191,7 @@ async fn test_branchall_nested(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4257,7 +4257,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 0) throw Error(JSON.stringify(l)); return { l: [...l, 0] } }", }, }, { @@ -4268,7 +4268,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 1) throw Error(JSON.stringify(l)); return { l: [...l, 1] } }", }, }, { @@ -4278,7 +4278,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 2) throw Error(JSON.stringify(l)); return { l: [...l, 2] } }", }, }], @@ -4286,7 +4286,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "value": { "input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(error) { return { 'from failure module': error } }", } }, @@ -4475,7 +4475,7 @@ fn hanging_step_value() -> serde_json::Value { serde_json::json!({ "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { await new Promise((r) => setTimeout(r, 600000)); }", }) } @@ -4487,7 +4487,7 @@ fn marker_failure_module() -> serde_json::Value { "value": { "input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(error) { return { handled_unrecoverable: true, error } }", } }) @@ -4585,7 +4585,7 @@ async fn test_unrecoverable_failure_on_continue_on_error_runs_failure_module( "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { ran_b: true }; }", }, }, @@ -4624,7 +4624,7 @@ async fn test_run_wait_result_early_return_with_failure_module( "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { throw new Error('boom'); }", }, @@ -4632,7 +4632,7 @@ async fn test_run_wait_result_early_return_with_failure_module( "failure_module": { "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { return { recovered: true } }", }, @@ -4720,7 +4720,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { throw new Error('boom'); }", }, }], @@ -4734,7 +4734,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { ran_b: true } }", }, }], @@ -4742,7 +4742,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { handled: true, recover: true } }", }, }, @@ -4797,14 +4797,23 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: server.close().await.unwrap(); #[cfg(feature = "quickjs")] - assert!(!chaining_error.success, "an error no handler recovered must keep the flow failed"); + assert!( + !chaining_error.success, + "an error no handler recovered must keep the flow failed" + ); - assert!(stopped.success, "a recovered failure inside a loop should end the flow as a success"); + assert!( + stopped.success, + "a recovered failure inside a loop should end the flow as a success" + ); assert!( stopped.json_result().unwrap().get("ran_b").is_none(), "the step after the loop must not run, as without recovery" ); - assert_eq!(stopped_handler_runs, 1, "the loop must stop at the first failed iteration"); + assert_eq!( + stopped_handler_runs, 1, + "the loop must stop at the first failed iteration" + ); assert!(skipping.success); assert_eq!( @@ -4815,7 +4824,11 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: assert_eq!(skipping_handler_runs, 3); assert!( - !parallel.json_result().unwrap().to_string().contains("ran_b"), + !parallel + .json_result() + .unwrap() + .to_string() + .contains("ran_b"), "the parallel loop fails as without recovery, so the step after it must not run" ); Ok(()) @@ -4987,6 +5000,101 @@ async fn test_flow_lock_all(db: Pool) -> anyhow::Result<()> { Ok(()) } +/// A deployed flow runs a multi-step loop body as the flow node its deploy stored, which +/// carries its own copy of the flow's `preserve_step_tags`. A preview never loads that node. +#[sqlx::test(fixtures("base"))] +async fn test_saved_flow_preserves_nested_step_tags(db: Pool) -> anyhow::Result<()> { + use futures::StreamExt; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + let flow = json!({ + "path": "g/all/saved_step_tags", "summary": "", "description": "", "tag": "python3", + "schema": { "type": "object", "properties": {}, "required": [] }, + "value": { + "preserve_step_tags": true, + "modules": [{ + "id": "loop", + "value": { + "type": "forloopflow", "skip_failures": false, + "iterator": { "type": "static", "value": [2, 3] }, + "modules": [{ + "id": "own_tag", + "value": { + "type": "rawscript", "language": "bun", "lock": "", "tag": "bun", + "content": "export function main(n: number) { return n * n; }", + "input_transforms": { + "n": { "type": "javascript", "expr": "flow_input.iter.value" } + } + } + }, { + "id": "untagged", + "value": { + "type": "rawscript", "language": "bun", "lock": "", + "content": "export function main(n: number) { return n + 1; }", + "input_transforms": { + "n": { "type": "javascript", "expr": "results.own_tag" } + } + } + }] + } + }] + } + }); + let mut completed = listen_for_completed_jobs(&db).await; + client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/flows/create" + )) + .bearer_auth("SECRET_TOKEN") + .json(&flow) + .send() + .await? + .error_for_status()?; + let deployment = in_test_worker(&db, completed.next(), port).await.unwrap(); + let deployed = completed_job(deployment, &db).await; + assert!(deployed.success, "{:?}", deployed.result); + + let job_id = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/f/g/all/saved_step_tags" + )) + .bearer_auth("SECRET_TOKEN") + .json(&json!({})) + .send() + .await? + .error_for_status()? + .text() + .await? + .parse::()?; + in_test_worker(&db, completed.find(&job_id), port).await; + let job = completed_job(job_id, &db).await; + assert!(job.success, "{:?}", job.result); + let tags = sqlx::query_as::<_, (String, String)>( + "WITH RECURSIVE descendants AS ( + SELECT id FROM v2_job WHERE id = $1 + UNION ALL + SELECT child.id FROM v2_job child JOIN descendants ON child.parent_job = descendants.id + ) + SELECT DISTINCT flow_step_id, tag FROM v2_job + WHERE id IN (SELECT id FROM descendants) AND flow_step_id IN ('own_tag', 'untagged') + ORDER BY flow_step_id, tag", + ) + .bind(job_id) + .fetch_all(&db) + .await?; + assert_eq!( + tags, + vec![ + ("own_tag".to_owned(), "bun".to_owned()), + ("untagged".to_owned(), "python3".to_owned()) + ] + ); + Ok(()) +} + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] @@ -5687,7 +5795,7 @@ async fn test_job_labels(db: Pool) -> anyhow::Result<()> { console.log(greet) return { greet, wm_labels: ["yolo", "greet", "greet", world] }; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -5896,7 +6004,7 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 42; }", "tag": "restricted-tag", }, @@ -5956,7 +6064,7 @@ async fn test_flow_substep_tag_checked_on_resolved_value(db: Pool) -> "id": id, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { lang: 'bun' } }", "tag": tag, }, diff --git a/backend/tests/worker_group_pause.rs b/backend/tests/worker_group_pause.rs new file mode 100644 index 0000000000..b528bbfd77 --- /dev/null +++ b/backend/tests/worker_group_pause.rs @@ -0,0 +1,71 @@ +//! In its own test binary: it flips the process-wide `WORKER_CONFIG`, which would stall every +//! other worker test sharing the process. + +use futures::StreamExt; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, + worker::{Connection, WORKER_CONFIG}, +}; +use windmill_test_utils::*; + +fn set_paused(paused: bool) { + let mut wc = (**WORKER_CONFIG.load()).clone(); + wc.paused = paused; + WORKER_CONFIG.store(std::sync::Arc::new(wc)); +} + +#[sqlx::test(fixtures("base"))] +async fn paused_worker_leaves_jobs_queued_until_resumed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + set_paused(true); + let id = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "echo done".to_string(), + path: None, + lock: None, + language: ScriptLang::Bash, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .push(&db) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker( + Connection::Sql(db.clone()), + async { + tokio::time::sleep(Duration::from_secs(3)).await; + let running = + sqlx::query_scalar::<_, bool>("SELECT running FROM v2_job_queue WHERE id = $1") + .bind(id) + .fetch_optional(&db) + .await + .unwrap(); + assert_eq!( + running, + Some(false), + "a paused worker must not pull the job" + ); + + set_paused(false); + while completed.next().await != Some(id) {} + }, + port, + ) + .await; + + assert!(completed_job(id, &db).await.success); + Ok(()) +} diff --git a/backend/tests/worker_ping_ip.rs b/backend/tests/worker_ping_ip.rs index 772f4193c3..2421e888ef 100644 --- a/backend/tests/worker_ping_ip.rs +++ b/backend/tests/worker_ping_ip.rs @@ -2,10 +2,19 @@ use sqlx::{Pool, Postgres}; use uuid::Uuid; use windmill_common::{ external_ip::UNKNOWN_IP, - worker::{insert_ping_query, update_worker_ping_main_loop_query}, + worker::{insert_ping_query, set_worker_draining_query, update_worker_ping_main_loop_query}, }; async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> anyhow::Result<()> { + register(db, worker, ip, None).await +} + +async fn register( + db: &Pool, + worker: &str, + ip: Option<&str>, + draining: Option, +) -> anyhow::Result<()> { insert_ping_query( "test-instance", worker, @@ -19,12 +28,33 @@ async fn insert_ping(db: &Pool, worker: &str, ip: Option<&str>) -> any None, None, false, + draining, db, ) .await?; Ok(()) } +/// A worker re-registers mid-life once its external IP resolves. That registration must not +/// clear a draining flag the same process already published; only a worker starting up under +/// a reclaimed name resets it. +#[sqlx::test] +async fn only_a_starting_worker_resets_draining(db: Pool) -> anyhow::Result<()> { + let draining = || async { + sqlx::query_scalar::<_, bool>("SELECT draining FROM worker_ping WHERE worker = 'wk-drain'") + .fetch_one(&db) + .await + }; + register(&db, "wk-drain", None, Some(false)).await?; + set_worker_draining_query("wk-drain", &db).await?; + register(&db, "wk-drain", Some("1.2.3.4"), None).await?; + assert!(draining().await?); + + register(&db, "wk-drain", None, Some(false)).await?; + assert!(!draining().await?); + Ok(()) +} + /// The external IP resolves in the background, so the initial ping often has none yet. That must /// not blank the address a previous process wrote to the row this one reclaims — worker names are /// stable across restarts under EXIT_AFTER_N_JOBS. diff --git a/backend/tests/zombie_flow_recovery.rs b/backend/tests/zombie_flow_recovery.rs index fb658dfb03..333aac70a8 100644 --- a/backend/tests/zombie_flow_recovery.rs +++ b/backend/tests/zombie_flow_recovery.rs @@ -78,7 +78,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children( "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -91,7 +91,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children( "id": "after", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -210,7 +210,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Res "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -223,7 +223,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Res "id": "after", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -331,7 +331,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "inner_first", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": {}, "content": "export function main() { return 'first' }" } @@ -339,7 +339,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "inner_second", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "first": { "type": "javascript", "expr": "results.inner_first" } }, @@ -353,7 +353,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "b": { "type": "javascript", "expr": "results.branch" } }, @@ -466,7 +466,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step( "modules": [{ "id": "inner", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -478,7 +478,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step( { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -578,7 +578,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Re "modules": [{ "id": "inner", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -590,7 +590,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Re { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, diff --git a/backend/windmill-ai/Cargo.toml b/backend/windmill-ai/Cargo.toml index b1dea3ae54..44c4ac83c7 100644 --- a/backend/windmill-ai/Cargo.toml +++ b/backend/windmill-ai/Cargo.toml @@ -28,6 +28,7 @@ eventsource-stream.workspace = true futures.workspace = true http.workspace = true mime_guess.workspace = true +rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/backend/windmill-ai/src/ai_bedrock.rs b/backend/windmill-ai/src/ai_bedrock.rs index a9318a1f3a..f55fcca415 100644 --- a/backend/windmill-ai/src/ai_bedrock.rs +++ b/backend/windmill-ai/src/ai_bedrock.rs @@ -477,6 +477,29 @@ pub async fn assume_role_with_oidc_token( // Error Formatting // ============================================================================ +/// The error for a Bedrock stream that failed after it opened, retrying the exceptions the +/// Vercel AI SDK retries (`amazon-bedrock-stream-error.ts`) and a broken connection. The +/// SDK's own retries cover the opening call alone. +pub fn bedrock_stream_error( + error: &aws_sdk_bedrockruntime::error::SdkError< + aws_sdk_bedrockruntime::types::error::ConverseStreamOutputError, + R, + >, +) -> Error { + let message = format!("Bedrock stream error: {}", format_bedrock_error(error)); + match error.as_service_error() { + Some(e) + if !(e.is_internal_server_exception() + || e.is_model_stream_error_exception() + || e.is_service_unavailable_exception() + || e.is_throttling_exception()) => + { + Error::AIError(message) + } + _ => crate::retry::transient_error(message, None), + } +} + /// Format AWS SDK errors with detailed information pub fn format_bedrock_error(error: &aws_sdk_bedrockruntime::error::SdkError) -> String where diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index bb8ae83e13..9f48edca8a 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -326,6 +326,17 @@ pub struct GeminiSSEEvent { pub candidates: Option>, #[serde(rename = "usageMetadata")] pub usage_metadata: Option, + #[serde(default)] + pub error: Option, + #[serde(rename = "promptFeedback", default)] + pub prompt_feedback: Option, +} + +/// Set when Gemini refuses the prompt itself, in which case no candidate follows. +#[derive(Deserialize, Debug)] +pub struct GeminiPromptFeedback { + #[serde(rename = "blockReason", default)] + pub block_reason: Option, } // ============================================================================ @@ -550,10 +561,40 @@ pub fn openai_tools_to_gemini( } } +/// Follows a Gemini stream to tell a complete turn from one cut short. +#[derive(Default)] +pub struct GeminiTurn { + ended: bool, + has_output: bool, +} + +impl GeminiTurn { + pub fn observe(&mut self, event: &GeminiParsedEvent) { + // A function call arrives whole and may come without a `finishReason`, so it + // counts too, as in gemini-cli's stream validation. + self.ended |= event.finish_reason.is_some() || !event.tool_calls.is_empty(); + self.has_output |= event.text.is_some() || event.reasoning.is_some(); + } + + /// Whether the turn's final event arrived. + pub fn ended(&self) -> bool { + self.ended + } + + /// Whether a stream that closed cleanly here holds the whole turn. Gemini 2.5 closes + /// its stream without a single event when thinking spends the whole + /// `maxOutputTokens`: an empty answer, which a retry would only get again, rather + /// than a cut one. + pub fn complete_at_close(&self) -> bool { + self.ended || !self.has_output + } +} + /// Parse one Gemini SSE data line into a [`GeminiParsedEvent`]. /// /// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`). /// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating. +/// An error the stream reports is returned as one. pub fn parse_gemini_sse_event(data: &str) -> Result, Error> { if data.is_empty() || data == "[DONE]" { return Ok(None); @@ -567,6 +608,21 @@ pub fn parse_gemini_sse_event(data: &str) -> Result, E } }; + if let Some(error) = event.error { + return Err(error.into_error("Gemini")); + } + + // `BLOCK_REASON_UNSPECIFIED` is the protobuf enum's placeholder, not a block. + if let Some(reason) = event + .prompt_feedback + .and_then(|f| f.block_reason) + .filter(|r| !matches!(r.as_str(), "" | "BLOCK_REASON_UNSPECIFIED")) + { + return Err(Error::AIError(format!( + "Gemini blocked the prompt: {reason}" + ))); + } + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; let Some(candidates) = event.candidates else { diff --git a/backend/windmill-ai/src/lib.rs b/backend/windmill-ai/src/lib.rs index ccc40da432..634614a3cf 100644 --- a/backend/windmill-ai/src/lib.rs +++ b/backend/windmill-ai/src/lib.rs @@ -11,6 +11,7 @@ pub mod model_context; pub mod providers; pub mod proxy; pub mod query_builder; +pub mod retry; pub mod sse; pub mod types; pub mod utils; diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 836244a666..5aa142a127 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -9,11 +9,12 @@ use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_bedrock::{ - bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop, - bedrock_stream_event_to_reasoning_delta, bedrock_stream_event_to_text, - bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_delta_with_block_index, - bedrock_stream_event_to_tool_start, bedrock_stream_event_to_tool_start_with_block_index, - build_tool_config, create_inference_config, format_bedrock_error, json_to_document, + bedrock_model_supports_prompt_caching, bedrock_stream_error, + bedrock_stream_event_is_block_stop, bedrock_stream_event_to_reasoning_delta, + bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, + bedrock_stream_event_to_tool_delta_with_block_index, bedrock_stream_event_to_tool_start, + bedrock_stream_event_to_tool_start_with_block_index, build_tool_config, + create_inference_config, format_bedrock_error, json_to_document, openai_messages_to_bedrock, streaming_tool_calls_to_openai, BearerTokenProvider, BedrockClient, StreamingToolCall, }, @@ -24,6 +25,7 @@ use crate::{ image_handler::prepare_messages_for_api, proxy::ProxyBuildArgs, query_builder::{ParsedResponse, StreamEventSink}, + retry::{openai_stream_error_chunk, truncated_stream_error}, types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef}, }; use bytes::Bytes; @@ -461,26 +463,31 @@ pub fn sdk_stream_to_sse( async_stream::stream! { let mut stream = stream; let mut state = BedrockSseStreamState::new(id, model, created); + let mut message_stopped = false; - loop { + let failure = loop { match stream.recv().await { Ok(Some(event)) => { + if matches!( + event, + aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_) + ) { + message_stopped = true; + } for chunk in bedrock_sse_chunks_for_event(&event, &mut state) { yield Ok(chunk); } } - Ok(None) => break, - Err(e) => { - yield Err(std::io::Error::new( - std::io::ErrorKind::Other, - e.to_string(), - )); - break; - } + Ok(None) => break (!message_stopped).then(truncated_stream_error), + Err(_) if message_stopped => break None, + Err(e) => break Some(bedrock_stream_error(&e)), } - } + }; - yield Ok(Bytes::from("data: [DONE]\n\n")); + match failure { + Some(error) => yield Ok(Bytes::from(openai_stream_error_chunk(&error))), + None => yield Ok(Bytes::from("data: [DONE]\n\n")), + } } } @@ -1081,11 +1088,19 @@ impl BedrockQueryBuilder { // Claude reasoning block for the turn (only populated when thinking is on), // attached to the first tool call for replay before toolUse. let mut reasoning: Option = None; + let mut message_stopped = false; // Process stream events using shared parsing functions loop { match stream.recv().await { Ok(Some(event)) => { + if matches!( + event, + aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_) + ) { + message_stopped = true; + } + // Fold reasoning deltas into the turn's reasoning block (for // replay before toolUse, required by Claude when thinking is // on) and stream the readable summary as a thinking affordance. @@ -1161,11 +1176,13 @@ impl BedrockQueryBuilder { } } Ok(None) => break, - Err(e) => { - return Err(Error::internal_err(format!("Bedrock stream error: {}", e))); - } + Err(_) if message_stopped => break, + Err(e) => return Err(bedrock_stream_error(&e)), } } + if !message_stopped { + return Err(truncated_stream_error()); + } // Send tool call events to stream processor if let Some(processor) = stream_event_sink.as_ref() { diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 97d0383ba4..bbf45e5e57 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -5,17 +5,20 @@ use crate::{ parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, GeminiPredictContent, GeminiTextRequest, - GeminiThinkingConfig, GeminiTool, + GeminiThinkingConfig, GeminiTool, GeminiTurn, }, image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, proxy::{ProxyBuildArgs, ProxyRequest}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, + retry::{ + openai_stream_error_chunk, send_with_retries, transient_error, truncated_stream_error, + }, sse::{GeminiSSEParser, SSEParser}, types::*, }; use async_trait::async_trait; use bytes::Bytes; -use eventsource_stream::Eventsource; +use eventsource_stream::{EventStreamError, Eventsource}; use futures::{stream::BoxStream, StreamExt}; use http::{header, HeaderMap, HeaderValue, Method, StatusCode}; use serde::Deserialize; @@ -336,9 +339,7 @@ pub async fn handle_google_ai_chat_proxy( ) -> Result { let GoogleAIProxyRequest { request, model, stream } = build_google_ai_chat_proxy_request(args)?; - let response = - send_google_ai_proxy_request(client, request, "Failed to send request to Gemini API") - .await?; + let response = send_google_ai_proxy_request(client, request).await?; if stream { Ok(convert_streaming_response(response, &model)) @@ -356,8 +357,7 @@ pub async fn handle_google_ai_models_proxy( args: &ProxyBuildArgs<'_>, ) -> Result { let request = build_google_ai_models_proxy_request(args); - let response = - send_google_ai_proxy_request(client, request, "Failed to fetch Gemini models").await?; + let response = send_google_ai_proxy_request(client, request).await?; let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| { Error::internal_err(format!("Failed to parse Gemini models response: {}", e)) @@ -505,26 +505,13 @@ fn add_google_ai_auth_header(headers: &mut Vec<(String, String)>, api_key: &str, async fn send_google_ai_proxy_request( client: &reqwest::Client, proxy_request: ProxyRequest, - send_error_message: &str, ) -> Result { let mut request = client.request(proxy_request.method.clone(), &proxy_request.url); for (header_name, header_value) in &proxy_request.headers { request = request.header(header_name.as_str(), header_value.as_str()); } - let response = request - .body(proxy_request.body) - .send() - .await - .map_err(|e| Error::internal_err(format!("{}: {}", send_error_message, e)))?; - - if let Err(e) = response.error_for_status_ref() { - let status = e.status().map(|s| s.to_string()).unwrap_or_default(); - let body = response.text().await.unwrap_or_default(); - return Err(Error::AIError(format!("{}: {}", status, body))); - } - - Ok(response) + send_with_retries(request.body(proxy_request.body)).await } fn convert_streaming_response(response: reqwest::Response, model: &str) -> GoogleAIProxyResponse { @@ -535,10 +522,15 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl let openai_sse_stream = async_stream::stream! { tokio::pin!(gemini_sse_stream); let mut tool_call_index: usize = 0; - while let Some(event) = gemini_sse_stream.next().await { + let mut turn = GeminiTurn::default(); + let failure = loop { + let Some(event) = gemini_sse_stream.next().await else { + break (!turn.complete_at_close()).then(truncated_stream_error); + }; match event { Ok(event) => match parse_gemini_sse_event(&event.data) { Ok(Some(parsed)) => { + turn.observe(&parsed); for chunk in gemini_event_to_openai_sse_chunks( &parsed, &id, &model, &mut tool_call_index, ) { @@ -546,12 +538,23 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl } } Ok(None) => {} - Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e), + Err(e) => break Some(e), }, + Err(EventStreamError::Transport(_)) if turn.ended() => break None, + Err(EventStreamError::Transport(e)) => { + break Some(transient_error( + format!("The connection to Gemini broke off mid-response: {e}"), + None, + )); + } Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e), } - } - yield Ok::(Bytes::from("data: [DONE]\n\n")); + }; + let last = match failure { + Some(error) => openai_stream_error_chunk(&error), + None => "data: [DONE]\n\n".to_string(), + }; + yield Ok::(Bytes::from(last)); } .boxed(); diff --git a/backend/windmill-ai/src/providers/mod.rs b/backend/windmill-ai/src/providers/mod.rs index e08b038bb0..7dcddc93a3 100644 --- a/backend/windmill-ai/src/providers/mod.rs +++ b/backend/windmill-ai/src/providers/mod.rs @@ -62,9 +62,13 @@ const REASONING_RULES: &[ReasoningRule] = &[ can_disable: false, completions_tools_need_off: false, }, - // Live-verified: astra takes low..max only, where sol and luna also take `none`. + // Live-verified: gpt-6-astra and gpt-6.1-sol take low..max only, where gpt-6-sol and + // gpt-6-luna also take `none`. ReasoningRule { - matches: |m| base_id(m).starts_with("gpt-6-astra"), + matches: |m| { + let base = base_id(m); + base.starts_with("gpt-6-astra") || base.starts_with("gpt-6.1-sol") + }, can_disable: false, completions_tools_need_off: true, }, diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index 1ea54a7e79..fea8edbdac 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -266,6 +266,7 @@ impl QueryBuilder for OtherQueryBuilder { mut events_str, stream_event_processor, usage: openai_usage, + .. } = openai_sse_parser; // Process streaming events with error handling diff --git a/backend/windmill-ai/src/retry.rs b/backend/windmill-ai/src/retry.rs new file mode 100644 index 0000000000..9de9d309da --- /dev/null +++ b/backend/windmill-ai/src/retry.rs @@ -0,0 +1,474 @@ +use std::time::Duration; + +use reqwest::{header::HeaderMap, RequestBuilder, Response, StatusCode}; +use windmill_common::error::{to_anyhow, Error}; + +/// How many times a provider call is retried after its first attempt. +pub const MAX_RETRIES: u32 = 3; +const BASE_DELAY: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(10); +/// A provider that asks for a longer wait is out of capacity for longer than anyone should +/// block on (a spent daily quota, say), so its error is returned instead of waited out. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(60); + +/// A provider call that failed for a reason a later attempt may not meet: the provider was +/// overloaded, throttled or unreachable, or its response broke off part way. +#[derive(Debug)] +pub struct TransientAIError { + pub message: String, + /// The wait the provider asked for, when it named one. + pub retry_after: Option, +} + +impl std::fmt::Display for TransientAIError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for TransientAIError {} + +pub fn transient_error(message: impl Into, retry_after: Option) -> Error { + to_anyhow(TransientAIError { message: message.into(), retry_after }).into() +} + +pub fn as_transient(error: &Error) -> Option<&TransientAIError> { + match error { + Error::Anyhow { error, .. } => error.downcast_ref::(), + _ => None, + } +} + +/// The error to report once a transient failure is no longer retried, without the source +/// location `Error::Anyhow` renders. +pub fn into_final_error(error: Error) -> Error { + match as_transient(&error) { + Some(transient) => Error::AIError(transient.message.clone()), + None => error, + } +} + +/// The statuses the OpenAI and Anthropic SDKs and the Vercel AI SDK retry. +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 409 | 429 | 500..=599) +} + +/// The error for a provider's error response. The rules follow the OpenAI and Anthropic +/// SDKs (`x-should-retry`, then the status) and gemini-cli (a daily quota is spent until +/// tomorrow, and the wait Gemini asks for is in its body). +pub fn response_error( + status: StatusCode, + headers: &HeaderMap, + body: &str, + message: String, +) -> Error { + let retryable = match headers.get("x-should-retry").and_then(|v| v.to_str().ok()) { + Some("true") => true, + Some("false") => false, + _ => { + is_retryable_status(status.as_u16()) + // OpenAI answers a spent balance with the 429 it also uses for rate limits. + && !body.contains("insufficient_quota") + && !google_quota_spent_for_the_day(body) + } + }; + if retryable { + transient_error( + message, + retry_after(headers).or_else(|| google_retry_delay(body)), + ) + } else { + Error::AIError(message) + } +} + +/// The `details` of a Google API error body, where Gemini puts its quota and retry hints. +fn google_error_details(body: &str) -> Vec { + serde_json::from_str::(body) + .ok() + .and_then(|v| v.get("error")?.get("details")?.as_array().cloned()) + .unwrap_or_default() +} + +fn google_quota_spent_for_the_day(body: &str) -> bool { + google_error_details(body).iter().any(|detail| { + detail + .get("violations") + .and_then(|v| v.as_array()) + .is_some_and(|violations| { + violations.iter().any(|violation| { + violation + .get("quotaId") + .and_then(|id| id.as_str()) + .is_some_and(|id| id.contains("PerDay") || id.contains("Daily")) + }) + }) + }) +} + +/// The `RetryInfo.retryDelay` of a Google API error, a protobuf duration such as `"53s"`. +fn google_retry_delay(body: &str) -> Option { + google_error_details(body).iter().find_map(|detail| { + let delay = detail.get("retryDelay")?.as_str()?.strip_suffix('s')?; + let seconds = delay + .parse::() + .ok() + .filter(|v| v.is_finite() && *v >= 0.0)?; + Some(Duration::try_from_secs_f64(seconds).unwrap_or(Duration::MAX)) + }) +} + +/// The wait a provider asked for in its `retry-after-ms` or `retry-after` header. An +/// HTTP-date `retry-after` is ignored, which falls back to the computed backoff. +fn retry_after(headers: &HeaderMap) -> Option { + let seconds = |name: &str, scale: f64| { + headers + .get(name)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .filter(|v| v.is_finite() && *v >= 0.0) + .map(|v| Duration::try_from_secs_f64(v / scale).unwrap_or(Duration::MAX)) + }; + seconds("retry-after-ms", 1000.0).or_else(|| seconds("retry-after", 1.0)) +} + +/// The error object a provider sends inside a stream: OpenAI-compatible chunks carry +/// `type` and a string or status `code`, Anthropic `type`, Gemini a status `code` and +/// `status`. +#[derive(Debug, Default, serde::Deserialize)] +pub struct StreamErrorBody { + #[serde(default)] + pub message: Option, + #[serde(default, rename = "type")] + pub kind: Option, + #[serde(default)] + pub code: Option, + #[serde(default)] + pub status: Option, +} + +impl StreamErrorBody { + /// The error for a failure an OpenAI-compatible or Gemini stream reported after opening + /// with a 200, classified as the Vercel AI SDK classifies OpenAI stream errors + /// (`openai-stream-error.ts`): the status `code` holds, else the status its code and type + /// name, else a server error. + pub fn into_error(self, provider: &str) -> Error { + let code = self.code_str(); + let discriminator = [code.as_deref(), self.kind.as_deref()] + .into_iter() + .flatten() + .collect::>() + .join(" ") + .to_lowercase(); + let has = |terms: &[&str]| terms.iter().any(|term| discriminator.contains(term)); + let status = match code.as_deref().and_then(|code| code.parse::().ok()) { + Some(status @ 400..=599) => status, + _ if has(&["insufficient_quota", "rate_limit"]) => 429, + _ if has(&["authentication"]) => 401, + _ if has(&["permission"]) => 403, + _ if has(&["not_found"]) => 404, + _ if has(&["invalid", "bad_request", "context_length"]) => 400, + _ if has(&["overload"]) => 503, + _ if has(&["timeout"]) => 504, + _ => 500, + }; + let retryable = !has(&["insufficient_quota"]) && is_retryable_status(status); + self.into_classified_error(provider, retryable) + } + + /// The error for a failure an Anthropic stream reported after opening with a 200. The + /// kinds retried are those the Vercel AI SDK retries, plus `timeout_error`, which + /// Anthropic documents as its 504; any other kind is a fault in the request or account. + pub fn into_anthropic_error(self) -> Error { + let retryable = matches!( + self.kind.as_deref(), + Some("api_error" | "overloaded_error" | "rate_limit_error" | "timeout_error") + ); + self.into_classified_error("Anthropic", retryable) + } + + fn code_str(&self) -> Option { + match self.code.as_ref()? { + serde_json::Value::String(code) => Some(code.clone()), + serde_json::Value::Number(code) => Some(code.to_string()), + _ => None, + } + } + + fn into_classified_error(self, provider: &str, retryable: bool) -> Error { + let kind = [self.kind.clone(), self.code_str(), self.status] + .into_iter() + .flatten() + .collect::>() + .join("/"); + let message = format!( + "{provider} stream error{}: {}", + (!kind.is_empty()) + .then(|| format!(" ({kind})")) + .unwrap_or_default(), + self.message.as_deref().unwrap_or("no details") + ); + if retryable { + transient_error(message, None) + } else { + Error::AIError(message) + } + } +} + +/// The error for a stream that closed before the provider sent the event ending a +/// complete response: cut off by the provider or by a proxy in between. +pub fn truncated_stream_error() -> Error { + transient_error( + "The AI provider's response stream ended before the response was complete", + None, + ) +} + +/// An OpenAI-format stream chunk reporting `error`, which ends a response converted to that +/// format after it started streaming: the chat's OpenAI SDK throws on a chunk carrying +/// `error`, where a bare end of stream would read as a finished answer. +pub fn openai_stream_error_chunk(error: &Error) -> String { + let message = as_transient(error).map_or_else(|| error.to_string(), |t| t.message.clone()); + format!( + "data: {}\n\n", + serde_json::json!({ "error": { "message": message } }) + ) +} + +/// Sends a provider request, retrying a transient failure while nothing of the response +/// has been read: a failed connection or an error status. Once retries are spent the error +/// carries the provider's answer, as a single attempt's would. +pub async fn send_with_retries(request: RequestBuilder) -> Result { + let mut backoff = Backoff::default(); + loop { + // A streamed body cannot be replayed, so such a request is sent once. + let Some(attempt) = request.try_clone() else { + return send_once(request).await.map_err(into_final_error); + }; + let error = match send_once(attempt).await { + Ok(response) => return Ok(response), + Err(error) => error, + }; + let Some(delay) = as_transient(&error).and_then(|t| backoff.next_delay(t)) else { + return Err(into_final_error(error)); + }; + tracing::warn!( + "AI provider call failed ({error}); retrying in {:.1}s ({}/{MAX_RETRIES})", + delay.as_secs_f64(), + backoff.retries() + ); + tokio::time::sleep(delay).await; + } +} + +/// The error for a provider request that failed before any response arrived. The request +/// timeout is the caller's whole budget, so a timed-out call leaves no time for another +/// attempt; a connect timeout has its own, shorter one. +pub fn send_error(e: reqwest::Error, context: &str) -> Error { + let message = format!("{context}: {e}"); + if e.is_builder() || (e.is_timeout() && !e.is_connect()) { + Error::AIError(message) + } else { + transient_error(message, None) + } +} + +async fn send_once(request: RequestBuilder) -> Result { + let response = request + .send() + .await + .map_err(|e| send_error(e, "Failed to reach the AI provider"))?; + let status = response.status(); + if status.is_client_error() || status.is_server_error() { + let headers = response.headers().clone(); + let body = response.text().await.unwrap_or_default(); + return Err(response_error(status, &headers, &body, body.clone())); + } + Ok(response) +} + +/// Counts the retries of one provider call and spaces them out. +#[derive(Debug, Default)] +pub struct Backoff { + retries: u32, +} + +impl Backoff { + /// The wait before retrying a call that failed with `error`, or `None` once it should + /// fail instead. + pub fn next_delay(&mut self, error: &TransientAIError) -> Option { + if self.retries >= MAX_RETRIES { + return None; + } + let delay = match error.retry_after { + Some(wait) if wait > MAX_RETRY_AFTER => return None, + // A zero wait would retry back to back, so it gets the computed backoff. + Some(wait) if !wait.is_zero() => wait, + // Jittered, so callers throttled together do not come back together. + _ => (BASE_DELAY * 2u32.pow(self.retries)) + .min(MAX_BACKOFF) + .mul_f64(0.5 + rand::random::() * 0.5), + }; + self.retries += 1; + Some(delay) + } + + /// Retries granted so far. + pub fn retries(&self) -> u32 { + self.retries + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Error responses as providers send them, each with the retry decision of the client + /// the rule is taken from. + #[test] + fn classifies_error_responses() { + const GEMINI_QUOTA: &str = r#"{"error":{"code":429,"status":"RESOURCE_EXHAUSTED","details":[{"@type":"type.googleapis.com/google.rpc.QuotaFailure","violations":[{"quotaId":"QUOTA_ID"}]},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"53s"}]}}"#; + let cases: &[(u16, &[(&str, &str)], &str, Option>)] = &[ + ( + 429, + &[], + r#"{"error":{"code":"rate_limit_exceeded"}}"#, + Some(None), + ), + ( + 429, + &[], + r#"{"error":{"type":"insufficient_quota","code":"insufficient_quota"}}"#, + None, + ), + ( + 529, + &[], + r#"{"type":"error","error":{"type":"overloaded_error"}}"#, + Some(None), + ), + (501, &[], "", Some(None)), + (400, &[], "", None), + (401, &[], "", None), + (400, &[("x-should-retry", "true")], "", Some(None)), + (503, &[("x-should-retry", "false")], "", None), + (429, &[("retry-after", "2")], "", Some(Some(2))), + ( + 429, + &[], + &GEMINI_QUOTA.replace("QUOTA_ID", "GenerateRequestsPerMinutePerProjectPerModel"), + Some(Some(53)), + ), + ( + 429, + &[], + &GEMINI_QUOTA.replace( + "QUOTA_ID", + "GenerateRequestsPerDayPerProjectPerModel-FreeTier", + ), + None, + ), + ]; + for (status, headers, body, expected) in cases { + let headers = headers + .iter() + .map(|(k, v)| (k.parse().unwrap(), v.parse().unwrap())) + .collect::(); + let error = response_error( + StatusCode::from_u16(*status).unwrap(), + &headers, + body, + String::new(), + ); + let got = as_transient(&error).map(|t| t.retry_after.map(|d| d.as_secs())); + assert_eq!(&got, expected, "{status} {headers:?} {body}"); + } + } + + /// In-stream error objects as providers send them, after the 200 opened the stream. + #[test] + fn classifies_stream_errors() { + let retried = |json: &str, anthropic: bool| { + let body: StreamErrorBody = serde_json::from_str(json).unwrap(); + let error = if anthropic { + body.into_anthropic_error() + } else { + body.into_error("AI provider") + }; + as_transient(&error).is_some() + }; + for (json, expected) in [ + (r#"{"message":"m","type":"server_error"}"#, true), + ( + r#"{"message":"m","type":"requests","code":"rate_limit_exceeded"}"#, + true, + ), + ( + r#"{"message":"m","type":"insufficient_quota","code":"insufficient_quota"}"#, + false, + ), + ( + r#"{"message":"m","type":"invalid_request_error","code":"context_length_exceeded"}"#, + false, + ), + // LiteLLM's proxy puts the upstream status in `code` as a string. + (r#"{"message":"m","type":"None","code":"400"}"#, false), + (r#"{"message":"m","code":"503"}"#, true), + (r#"{"code":503,"message":"m","status":"UNAVAILABLE"}"#, true), + ( + r#"{"code":400,"message":"m","status":"INVALID_ARGUMENT"}"#, + false, + ), + (r#"{"message":"m"}"#, true), + ] { + assert_eq!(retried(json, false), expected, "{json}"); + } + for (kind, expected) in [ + ("overloaded_error", true), + ("api_error", true), + ("rate_limit_error", true), + ("timeout_error", true), + ("billing_error", false), + ("invalid_request_error", false), + ("some_new_error", false), + ] { + let json = format!(r#"{{"type":"{kind}","message":"m"}}"#); + assert_eq!(retried(&json, true), expected, "{kind}"); + } + } + + #[test] + fn honours_retry_after_up_to_a_bound() { + let mut headers = HeaderMap::new(); + headers.insert("retry-after", "2".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::from_secs(2))); + headers.insert("retry-after-ms", "1500".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::from_millis(1500))); + headers.remove("retry-after-ms"); + headers.insert("retry-after", "1e30".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::MAX)); + + let mut backoff = Backoff::default(); + let asked = |secs| TransientAIError { + message: String::new(), + retry_after: Some(Duration::from_secs(secs)), + }; + assert_eq!(backoff.next_delay(&asked(5)), Some(Duration::from_secs(5))); + assert_eq!(backoff.next_delay(&asked(3600)), None); + } + + #[test] + fn stops_after_max_retries() { + let mut backoff = Backoff::default(); + let error = TransientAIError { message: String::new(), retry_after: None }; + for _ in 0..MAX_RETRIES { + let delay = backoff.next_delay(&error).expect("retry"); + assert!(delay <= MAX_BACKOFF); + } + assert_eq!(backoff.next_delay(&error), None); + } +} diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index ada50afca5..bdaa75025f 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use eventsource_stream::Eventsource; +use eventsource_stream::{EventStreamError, Eventsource}; use indexmap::IndexMap; use reqwest::Response; use serde::Deserialize; @@ -8,12 +8,13 @@ use tokio_stream::StreamExt; use windmill_common::{error::Error, utils::rd_string}; use crate::{ - ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, + ai_google::{parse_gemini_sse_event, GeminiTurn, GeminiUsageMetadata}, ai_types::UrlCitation, ai_types::{ AnthropicExtraContent, ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall, }, query_builder::StreamEventSink, + retry::{transient_error, truncated_stream_error, StreamErrorBody}, types::{StreamingEvent, TokenUsage}, }; @@ -43,6 +44,8 @@ pub struct OpenAIChoiceDelta { #[derive(Deserialize)] pub struct OpenAIChoice { pub delta: Option, + #[serde(default)] + pub finish_reason: Option, } /// Nested prompt token details returned by the Chat Completions API. @@ -87,6 +90,8 @@ pub struct OpenAISSEEvent { pub choices: Option>, #[serde(default)] pub usage: Option, + #[serde(default)] + pub error: Option, } lazy_static::lazy_static! { @@ -99,6 +104,14 @@ lazy_static::lazy_static! { pub trait SSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; + /// Whether the stream delivered the event that ends a complete response. + fn is_complete(&self) -> bool; + + /// Whether a stream that closed cleanly holds a complete response. + fn is_complete_at_close(&self) -> bool { + self.is_complete() + } + async fn parse_events(&mut self, response: Response) -> Result<(), Error> { let mut stream = response.bytes_stream().eventsource(); let mut consecutive_errors = 0; @@ -113,6 +126,18 @@ pub trait SSEParser { self.parse_event_data(&event.data).await?; } + // The body ends right after a transport error, so it cannot be skipped + // like a malformed event. A timeout spent the caller's whole budget. + Err(EventStreamError::Transport(_)) if self.is_complete() => break, + Err(EventStreamError::Transport(e)) => { + let message = + format!("The connection to the AI provider broke off mid-response: {e}"); + return Err(if e.is_timeout() { + Error::AIError(message) + } else { + transient_error(message, None) + }); + } Err(e) => { consecutive_errors += 1; tracing::error!("Failed to parse SSE event: {}", e); @@ -130,6 +155,9 @@ pub trait SSEParser { } } + if !self.is_complete_at_close() { + return Err(truncated_stream_error()); + } Ok(()) } } @@ -145,6 +173,9 @@ pub struct OpenAISSEParser { pub stream_event_processor: Box, /// Token usage from final chunk (when stream_options.include_usage is true) pub usage: Option, + /// Set by `[DONE]` or by a choice's `finish_reason`: OpenAI-compatible gateways do not + /// all send `[DONE]`. + complete: bool, } impl OpenAISSEParser { @@ -156,13 +187,19 @@ impl OpenAISSEParser { events_str: String::new(), stream_event_processor, usage: None, + complete: false, } } } impl SSEParser for OpenAISSEParser { + fn is_complete(&self) -> bool { + self.complete + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { if data == "[DONE]" { + self.complete = true; return Ok(()); } @@ -173,13 +210,21 @@ impl SSEParser for OpenAISSEParser { .ok(); if let Some(event) = event { + if let Some(error) = event.error { + return Err(error.into_error("AI provider")); + } + // Extract usage from final chunk (when stream_options.include_usage is true) if let Some(usage) = event.usage { self.usage = Some(usage); } if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { - if let Some(delta) = choices.remove(0).delta { + let choice = choices.remove(0); + if choice.finish_reason.is_some() { + self.complete = true; + } + if let Some(delta) = choice.delta { if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) { self.accumulated_reasoning.push_str(&reasoning); let event = StreamingEvent::ReasoningTokenDelta { content: reasoning }; @@ -345,6 +390,8 @@ pub enum AnthropicSSEEvent { MessageDelta { #[serde(default)] usage: Option, + #[serde(default)] + delta: Option, }, #[serde(rename = "message_stop")] MessageStop {}, @@ -353,12 +400,18 @@ pub enum AnthropicSSEEvent { #[serde(rename = "error")] Error { #[serde(default)] - message: Option, + error: StreamErrorBody, }, #[serde(other)] Unknown, } +#[derive(Deserialize, Debug)] +pub struct AnthropicMessageDelta { + #[serde(default)] + pub stop_reason: Option, +} + /// Tracks state of a content block during streaming #[derive(Debug)] #[allow(dead_code)] @@ -391,6 +444,9 @@ pub struct AnthropicSSEParser { /// can be replayed before `tool_use` (required by Claude when thinking is on). pending_reasoning: Option, reasoning_attached: bool, + /// Set by `message_stop` or by the `stop_reason` of the `message_delta` sent just + /// before it, so a gateway that drops `message_stop` does not fail a whole answer. + complete: bool, } impl AnthropicSSEParser { @@ -407,11 +463,16 @@ impl AnthropicSSEParser { usage: None, pending_reasoning: None, reasoning_attached: false, + complete: false, } } } impl SSEParser for AnthropicSSEParser { + fn is_complete(&self) -> bool { + self.complete + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let event: Option = serde_json::from_str(data) .inspect_err(|e| { @@ -583,9 +644,8 @@ impl SSEParser for AnthropicSSEParser { .await?; } } - AnthropicSSEEvent::Error { message } => { - let error_msg = message.unwrap_or_else(|| "Unknown error".to_string()); - tracing::error!("Anthropic streaming error: {}", error_msg); + AnthropicSSEEvent::Error { error } => { + return Err(error.into_anthropic_error()); } AnthropicSSEEvent::MessageStart { message } => { // The only event carrying the prompt-side counts. `message_delta` @@ -595,7 +655,10 @@ impl SSEParser for AnthropicSSEParser { self.usage = Some(usage); } } - AnthropicSSEEvent::MessageDelta { usage } => { + AnthropicSSEEvent::MessageDelta { usage, delta } => { + if delta.is_some_and(|d| d.stop_reason.is_some()) { + self.complete = true; + } if let Some(u) = usage { match &mut self.usage { // Field by field, so the prompt counts from `message_start` @@ -614,10 +677,9 @@ impl SSEParser for AnthropicSSEParser { } } } + AnthropicSSEEvent::MessageStop {} => self.complete = true, // Ignore other events - AnthropicSSEEvent::MessageStop {} - | AnthropicSSEEvent::Ping {} - | AnthropicSSEEvent::Unknown => {} + AnthropicSSEEvent::Ping {} | AnthropicSSEEvent::Unknown => {} } } @@ -645,6 +707,7 @@ pub struct GeminiSSEParser { pub annotations: Vec, pub used_websearch: bool, pub usage: Option, + turn: GeminiTurn, } impl GeminiSSEParser { @@ -659,15 +722,25 @@ impl GeminiSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + turn: GeminiTurn::default(), } } } impl SSEParser for GeminiSSEParser { + fn is_complete(&self) -> bool { + self.turn.ended() + } + + fn is_complete_at_close(&self) -> bool { + self.turn.complete_at_close() + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let Some(parsed) = parse_gemini_sse_event(data)? else { return Ok(()); }; + self.turn.observe(&parsed); if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) { self.accumulated_reasoning.push_str(&reasoning); @@ -797,6 +870,8 @@ impl OpenAIResponsesUsage { pub struct OpenAIResponsesResponse { #[serde(default)] pub usage: Option, + #[serde(default)] + pub error: Option, } /// SSE event types for OpenAI Responses API streaming @@ -833,6 +908,20 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.incomplete")] Incomplete { response: OpenAIResponsesResponse }, + #[serde(rename = "response.failed")] + Failed { response: OpenAIResponsesResponse }, + + /// Flat (`code`, `message`) from OpenAI, nested under `error` from some compatible servers. + #[serde(rename = "error")] + Error { + #[serde(default)] + code: Option, + #[serde(default)] + message: Option, + #[serde(default)] + error: Option, + }, + /// Response created #[serde(rename = "response.created")] Created {}, @@ -890,6 +979,7 @@ pub struct OpenAIResponsesSSEParser { pub usage: Option, /// Reasoning summary parts seen so far, to separate them as paragraphs reasoning_summary_parts: usize, + response_ended: bool, } impl OpenAIResponsesSSEParser { @@ -906,11 +996,16 @@ impl OpenAIResponsesSSEParser { used_websearch: false, usage: None, reasoning_summary_parts: 0, + response_ended: false, } } } impl SSEParser for OpenAIResponsesSSEParser { + fn is_complete(&self) -> bool { + self.response_ended + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let event: Option = serde_json::from_str(data) .inspect_err(|e| { @@ -1010,11 +1105,22 @@ impl SSEParser for OpenAIResponsesSSEParser { OpenAIResponsesSSEEvent::Completed { response } | OpenAIResponsesSSEEvent::Incomplete { response } => { + self.response_ended = true; if let Some(usage) = response.usage { self.usage = Some(usage); } } + OpenAIResponsesSSEEvent::Failed { response } => { + return Err(response.error.unwrap_or_default().into_error("OpenAI")); + } + + OpenAIResponsesSSEEvent::Error { code, message, error } => { + return Err(error + .unwrap_or(StreamErrorBody { code, message, ..Default::default() }) + .into_error("OpenAI")); + } + OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => { self.reasoning_summary_parts += 1; if self.reasoning_summary_parts > 1 { @@ -1037,9 +1143,10 @@ impl SSEParser for OpenAIResponsesSSEParser { } } + OpenAIResponsesSSEEvent::Done {} => self.response_ended = true, + // Ignore other event types - OpenAIResponsesSSEEvent::Done {} - | OpenAIResponsesSSEEvent::Created {} + OpenAIResponsesSSEEvent::Created {} | OpenAIResponsesSSEEvent::InProgress {} | OpenAIResponsesSSEEvent::OutputItemDone {} | OpenAIResponsesSSEEvent::ContentPartAdded {} @@ -1093,6 +1200,117 @@ mod tests { } } + fn sse_response(chunks: Vec>) -> Response { + let body = reqwest::Body::wrap_stream(futures::stream::iter(chunks)); + Response::from(http::Response::new(body)) + } + + /// A stream that breaks off, or reports a failure inside its 200, must fail the call + /// rather than hand back what arrived as if it were the whole answer. + #[tokio::test] + async fn stream_failures_are_errors_not_partial_answers() { + use crate::retry::as_transient; + + const STARTED: &str = concat!( + "data: {\"type\":\"message_start\"}\n\n", + "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n", + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hel\"}}\n\n", + ); + let anthropic = |chunks| async move { + AnthropicSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(chunks)) + .await + }; + + let overloaded = anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n"), + ]) + .await + .unwrap_err(); + assert!(as_transient(&overloaded).is_some(), "{overloaded}"); + + let refused = anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"bad\"}}\n\n"), + ]) + .await + .unwrap_err(); + assert!(as_transient(&refused).is_none(), "{refused}"); + + let truncated = anthropic(vec![Ok(STARTED)]).await.unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + + let dropped = anthropic(vec![ + Ok(STARTED), + Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + )), + ]) + .await + .unwrap_err(); + assert!(as_transient(&dropped).is_some(), "{dropped}"); + + anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"message_stop\"}\n\n"), + ]) + .await + .unwrap(); + + anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n"), + ]) + .await + .unwrap(); + + let truncated = OpenAISSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n", + )])) + .await + .unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + + let quota = OpenAIResponsesSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"type\":\"error\",\"error\":{\"type\":\"insufficient_quota\",\"code\":\"insufficient_quota\",\"message\":\"m\"}}\n\n", + )])) + .await + .unwrap_err(); + assert!(as_transient("a).is_none(), "{quota}"); + + GeminiSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"name\":\"get_weather\",\"args\":{}}}]}}]}\n\n", + )])) + .await + .unwrap(); + + let gemini = |chunks| async move { + GeminiSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(chunks)) + .await + }; + // Thinking spent the whole `maxOutputTokens`: a clean close with no event. + gemini(vec![]).await.unwrap(); + let dropped = gemini(vec![Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + ))]) + .await + .unwrap_err(); + assert!(as_transient(&dropped).is_some(), "{dropped}"); + let truncated = gemini(vec![Ok( + "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Hi\"}]}}]}\n\n", + )]) + .await + .unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + } + /// The prompt-side counts arrive only on `message_start` and the completion total /// only on `message_delta`; a parser that keeps just the last one reports a request /// with no input tokens at all. diff --git a/backend/windmill-ai/src/types.rs b/backend/windmill-ai/src/types.rs index e591e85159..59733e352c 100644 --- a/backend/windmill-ai/src/types.rs +++ b/backend/windmill-ai/src/types.rs @@ -1,3 +1,4 @@ +use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::collections::HashMap; @@ -533,7 +534,7 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none")] pub items: Option>, #[serde(skip_serializing_if = "Option::is_none")] - pub properties: Option>>, + pub properties: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub required: Option>, #[serde(skip_serializing_if = "Option::is_none", rename = "oneOf")] @@ -560,9 +561,9 @@ pub struct OpenAPISchema { #[serde(skip_serializing_if = "Option::is_none", rename = "$ref")] pub ref_path: Option, #[serde(skip_serializing_if = "Option::is_none", rename = "$defs")] - pub defs: Option>>, + pub defs: Option>>, #[serde(skip_serializing_if = "Option::is_none")] - pub definitions: Option>>, + pub definitions: Option>>, // Schema composition #[serde(skip_serializing_if = "Option::is_none", rename = "allOf")] @@ -829,7 +830,7 @@ impl OpenAPISchema { // Merge properties (other's properties are added if key doesn't exist) if let Some(ref other_props) = other.properties { - let props = self.properties.get_or_insert_with(HashMap::new); + let props = self.properties.get_or_insert_with(IndexMap::new); for (key, value) in other_props { props.entry(key.clone()).or_insert_with(|| value.clone()); } @@ -880,13 +881,13 @@ impl OpenAPISchema { // Merge $defs and definitions if let Some(ref other_defs) = other.defs { - let defs = self.defs.get_or_insert_with(HashMap::new); + let defs = self.defs.get_or_insert_with(IndexMap::new); for (key, value) in other_defs { defs.entry(key.clone()).or_insert_with(|| value.clone()); } } if let Some(ref other_definitions) = other.definitions { - let definitions = self.definitions.get_or_insert_with(HashMap::new); + let definitions = self.definitions.get_or_insert_with(IndexMap::new); for (key, value) in other_definitions { definitions .entry(key.clone()) @@ -981,7 +982,29 @@ pub struct S3ObjectWithType { #[cfg(test)] mod tests { use super::*; - use std::collections::HashMap; + + /// The schema is part of the prompt prefix the provider caches on, so its keys must + /// serialize in the same order on every request. + #[test] + fn strict_schema_keeps_the_defined_property_order() { + let keys = [ + "zeta", "alpha", "mid", "beta", "omega", "gamma", "delta", "kappa", + ]; + let props = keys + .iter() + .map(|k| format!(r#""{k}":{{"type":"string"}}"#)) + .collect::>() + .join(","); + let mut schema: OpenAPISchema = + serde_json::from_str(&format!(r#"{{"type":"object","properties":{{{props}}}}}"#)) + .unwrap(); + schema.make_strict(); + + let value = serde_json::to_value(&schema).unwrap(); + let serialized: Vec<&String> = value["properties"].as_object().unwrap().keys().collect(); + assert_eq!(serialized, keys); + assert_eq!(value["required"], serde_json::json!(keys)); + } /// Whichever provider served the request, `input_tokens` has to end up meaning the /// whole prompt. Leaving the Anthropic shape as reported under-states it by the @@ -1248,7 +1271,7 @@ mod tests { #[test] fn test_make_strict_defs() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1274,7 +1297,7 @@ mod tests { #[test] fn test_make_strict_definitions() { let def_schema = object_schema(vec![("prop", string_schema())]); - let mut definitions = HashMap::new(); + let mut definitions = IndexMap::new(); definitions.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { definitions: Some(definitions), ..Default::default() }; @@ -1528,7 +1551,7 @@ mod tests { #[test] fn test_flatten_all_of_preserves_defs() { let def_schema = object_schema(vec![("field", string_schema())]); - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let schema_with_defs = OpenAPISchema { defs: Some(defs), ..Default::default() }; @@ -1642,7 +1665,7 @@ mod tests { schema_url: Some("http://json-schema.org/draft-07/schema#".to_string()), ..Default::default() }; - let mut defs = HashMap::new(); + let mut defs = IndexMap::new(); defs.insert("MyType".to_string(), Box::new(def_schema)); let mut schema = OpenAPISchema { diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index dce6a0ff72..21f916bf94 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -170,10 +170,16 @@ const FLOW_JOBS: [&'static str; 6] = [ /// scoped by the agent's path like a deployed runnable's, as `jobs:run:agents:`. const AGENT_JOBS: [&'static str; 1] = ["jobs/run/agent"]; +/// Lock resolution for scripts, apps and flows, scoped as `jobs:run:dependencies:`. +/// Resolving a package can run its build or install scripts on the worker, so this is +/// still code execution: the scope only takes away the direct ways to run code. +const DEPENDENCY_JOBS: [&'static str; 2] = ["jobs/run/dependencies", "jobs/run/flow_dependencies"]; + lazy_static::lazy_static! { static ref RUN_PATH_ACTIONS: Vec<&'static str> = { - let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component", "apps_u/upload_s3_file"]; + let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "apps_u/execute_component", "apps_u/upload_s3_file"]; + v.extend(DEPENDENCY_JOBS); v.extend(SCRIPT_JOBS); v.extend(FLOW_JOBS); v.extend(AGENT_JOBS); @@ -227,11 +233,13 @@ fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { } } -/// Checks the route path to determine the runnable kind ("agents", "flows" or "scripts"). +/// Checks the route path to determine the runnable kind ("dependencies", "agents", "flows" +/// or "scripts"). /// /// The order of checks is important: -/// - Flow-related paths are checked first to avoid false positives, as some flow paths -/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). +/// - Dependency paths are checked before flow paths: `flow_dependencies` starts with `f`. +/// - Flow-related paths are checked before script paths to avoid false positives, as some +/// flow paths (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). /// /// Returns the kind based on the match, or `None` if no match is found. fn determine_kind_from_route(route_path: &str) -> Option { @@ -247,6 +255,12 @@ fn determine_kind_from_route(route_path: &str) -> Option { { return None; } + if DEPENDENCY_JOBS + .iter() + .any(|path| route_path.starts_with(path)) + { + return Some("dependencies".to_string()); + } if AGENT_JOBS.iter().any(|path| route_path.starts_with(path)) { return Some("agents".to_string()); } @@ -586,17 +600,23 @@ pub struct JobReadConfinement { } impl JobReadConfinement { - /// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows` or `agents`) - /// is inside the confinement. A read path admits every kind; a run scope only its own. + /// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows`, `agents` or + /// `dependencies`) is inside the confinement. A read path admits every kind; a run + /// scope only its own. pub fn admits(&self, kind: &str, runnable_path: &str) -> bool { if self.read_paths.allows(runnable_path) { return true; } + // Lock jobs and previews store the path their caller sent. The matcher reads `*` + // in it as a pattern, and `,`/`:` are scope syntax: such a path needs a grant + // covering every path, as enqueuing it did. + let resource = windmill_common::auth::is_scope_literal_path(runnable_path) + .then(|| vec![runnable_path.to_string()]); let required = ScopeDefinition::new( ScopeDomain::Jobs.as_str(), ScopeAction::Run.as_str(), Some(kind), - Some(vec![runnable_path.to_string()]), + resource, ); self.run.iter().any(|scope| scope.includes(&required)) } @@ -862,7 +882,8 @@ fn scope_grants_access( /// from its own super-admin gate, these among them, so each is already readable by every /// authenticated user; each is here because the CLI reads it from a job: /// `automate_username_creation` before creating a user during a git-sync push, `uid` and -/// `hub_base_url` when `u/admin/hub_sync` pulls resource types from the Hub. The other +/// `hub_base_url` when a job runs `wmill hub pull`, as an edited copy of the retired +/// `u/admin/hub_sync` still does (servers sync resource types themselves). The other /// ungated keys have no such caller, so they stay confined — being ungated earns a key /// nothing on its own. Listing a gated key earns it nothing either: `require_super_admin` /// refuses every job token, so `hub_api_secret`, which that pull reads for a private Hub, @@ -996,6 +1017,7 @@ pub fn is_own_job_runtime_route(route_path: &str, http_method: &str, job_id: uui ["job_metrics", "set_progress", ..] => (http_method == "POST", 2), ["job_metrics", "get_progress", ..] => (http_method == "GET", 2), ["jobs" | "jobs_u", "resume_urls" | "wac_approval_urls", ..] => (http_method == "GET", 2), + ["jobs", "worker_is_draining", ..] => (http_method == "GET", 2), ["jobs", "wac", "inline_checkpoint", ..] => (http_method == "POST", 3), ["jobs", "run", "workflow_as_code", ..] => (http_method == "POST", 3), _ => return false, @@ -1713,6 +1735,24 @@ mod tests { scope_for_route("POST", "/api/w/ws/jobs/run/agent/u/x/y").as_deref(), Some("jobs:run:agents") ); + for path in [ + "/api/w/ws/jobs/run/dependencies", + "/api/w/ws/jobs/run/dependencies_async", + "/api/w/ws/jobs/run/flow_dependencies", + "/api/w/ws/jobs/run/flow_dependencies_async", + ] { + assert_eq!( + scope_for_route("POST", path).as_deref(), + Some("jobs:run:dependencies"), + "{path}" + ); + assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_ok()); + assert!(check_route_access(&["jobs:run:flows".into()], path, "POST").is_err()); + } + // A dependencies grant reaches no other run route. + for path in ["/api/w/ws/jobs/run/preview", "/api/w/ws/jobs/run/p/u/x/y"] { + assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_err()); + } // Preview/bundle runs have no deployed path and their handlers require the // broad `jobs:run` scope, so the derived scope must not carry a kind. diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 7598021f00..f1f4792e71 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -224,12 +224,13 @@ async fn update_config( #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { - // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode + // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode and paused serde_json::json!({ "worker_tags": config.get("worker_tags"), "cache_clear": config.get("cache_clear"), "init_bash": config.get("init_bash"), - "native_mode": config.get("native_mode") + "native_mode": config.get("native_mode"), + "paused": config.get("paused") }) } else { config @@ -272,6 +273,18 @@ async fn update_config( } let mut tx = db.begin().await?; + let was_paused = if name.starts_with("worker__") { + sqlx::query_scalar!( + "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + &name + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .unwrap_or(false) + } else { + false + }; sqlx::query!( "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config", &name, @@ -291,6 +304,17 @@ async fn update_config( ) .await?; tx.commit().await?; + + if name.starts_with("worker__") { + let paused = config.get("paused").and_then(|v| v.as_bool()) == Some(true); + if paused != was_paused { + windmill_common::feature_usage::log_feature_usage( + "worker_group", + "pause_toggle", + if paused { "pause" } else { "resume" }, + ); + } + } Ok(format!("Updated config {name}")) } diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 1007cfd5b1..eab73bc015 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -20,7 +20,8 @@ use windmill_api_auth::{ require_owner_of_path, ApiAuthed, }; use windmill_common::workspaces::{ - check_deploy_rules, check_operator_can_build_flows, operator_can_build_flows, RuleCheckResult, + check_deploy_rules, check_operator_can_build, operator_builder_rights, BuilderKind, + RuleCheckResult, }; use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, @@ -243,7 +244,7 @@ async fn list_flows( // Append the authed user's drafts at paths with no deployed flow; see scripts.rs. if lq.include_draft_only.unwrap_or(false) - && (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?) + && (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.flows) && offset == 0 && lq.path_start.is_none() && lq.path_exact.is_none() @@ -731,7 +732,14 @@ async fn create_flow( Path(w_id): Path, Json(mut nf): Json, ) -> Result<(StatusCode, String)> { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "create flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "create flows", + ) + .await?; check_scopes(&authed, || format!("flows:write:{}", nf.path))?; // A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly. @@ -1344,7 +1352,14 @@ async fn update_flow( Path((w_id, flow_path)): Path<(String, StripPath)>, Json(ef): Json, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "update flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "update flows", + ) + .await?; let flow_path = flow_path.to_path(); // The URL identifies the flow being updated; the body path is only needed to rename. let mut nf = ef.into_new_flow(flow_path); @@ -2038,7 +2053,14 @@ async fn archive_flow_by_path( Path((w_id, path)): Path<(String, StripPath)>, Json(archived): Json, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "archive flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "archive flows", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("flows:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( @@ -2179,7 +2201,14 @@ async fn delete_flow_by_path( Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "delete flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "delete flows", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("flows:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( diff --git a/backend/windmill-api-groups/src/groups.rs b/backend/windmill-api-groups/src/groups.rs index 0209af8585..d9550d2f5b 100644 --- a/backend/windmill-api-groups/src/groups.rs +++ b/backend/windmill-api-groups/src/groups.rs @@ -470,26 +470,16 @@ async fn update_igroup( require_super_admin(&db, &authed).await?; let mut tx: Transaction<'_, Postgres> = db.begin().await?; - let exists_opt = sqlx::query("SELECT 1 FROM instance_group WHERE name = $1") - .bind(name.clone()) - .fetch_optional(&mut *tx) - .await?; - not_found_if_none(exists_opt, "instance_group", name.clone())?; + let old_role_opt = sqlx::query_scalar!( + "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", + &name + ) + .fetch_optional(&mut *tx) + .await?; + let old_role = not_found_if_none(old_role_opt, "instance_group", name.clone())?; let validated_role = validate_instance_role(&igroup_update.instance_role)?; - // Fetch old role before updating so we can detect changes - let old_role = if igroup_update.instance_role.is_some() { - sqlx::query_scalar!( - "SELECT instance_role FROM instance_group WHERE name = $1", - &name - ) - .fetch_one(&mut *tx) - .await? - } else { - None - }; - sqlx::query("UPDATE instance_group SET summary = $1, instance_role = $2 WHERE name = $3") .bind(igroup_update.new_summary) .bind(&validated_role) @@ -497,8 +487,9 @@ async fn update_igroup( .execute(&mut *tx) .await?; - // If instance_role actually changed, propagate to all group members - if igroup_update.instance_role.is_some() && old_role != validated_role { + // An omitted or null instance_role is stored as NULL above, so it revokes like any other + // change: members must be recomputed whenever the stored role differs. + if old_role != validated_role { propagate_instance_group_roles(&name, &mut tx).await?; } @@ -664,21 +655,19 @@ async fn delete_igroup( // FOR UPDATE: the group row is the group-level mutex, taken before the workspace // advisory locks (see reconcile_workspace_instance_groups). - let group_role = sqlx::query_scalar!( + sqlx::query_scalar!( "SELECT instance_role FROM instance_group WHERE name = $1 FOR UPDATE", &name ) .fetch_optional(&mut *tx) - .await? - .flatten(); + .await?; - let affected_members: Vec = if group_role.is_some() { + // Collected whatever the group's current role: a member can still hold a role this group + // granted before it was cleared. + let affected_members: Vec = sqlx::query_scalar!("SELECT email FROM email_to_igroup WHERE igroup = $1", &name) .fetch_all(&mut *tx) - .await? - } else { - vec![] - }; + .await?; // Captured and advisory-locked before the settings update strips the group from them. #[cfg(feature = "private")] diff --git a/backend/windmill-api-integration-tests/tests/datatable_roles.rs b/backend/windmill-api-integration-tests/tests/datatable_roles.rs index ddb94f8c2b..023d9f20ed 100644 --- a/backend/windmill-api-integration-tests/tests/datatable_roles.rs +++ b/backend/windmill-api-integration-tests/tests/datatable_roles.rs @@ -417,6 +417,89 @@ async fn concurrent_role_creations_both_survive(db: Pool) -> anyhow::R outcome } +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn an_existing_role_is_taken_over_only_when_asked_and_never_with_members( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let plain = format!("wmtest_plain_{suffix}"); + let group = format!("wmtest_group_{suffix}"); + let member = format!("wmtest_member_{suffix}"); + for sql in [ + // An expired password and no connection slot: kept as is, the login Windmill sets is dead. + format!( + "CREATE ROLE \"{plain}\" LOGIN PASSWORD 'theirs' VALID UNTIL '2000-01-01' \ + CONNECTION LIMIT 0" + ), + format!("CREATE ROLE \"{group}\" NOLOGIN"), + format!("CREATE ROLE \"{member}\" LOGIN"), + format!("GRANT \"{group}\" TO \"{member}\""), + ] { + sqlx::query(&sql).execute(&db).await?; + } + + let create = |name: String, take_over: bool| async move { + let resp = authed( + client().post(format!( + "http://localhost:{port}/api/settings/datatable_roles" + )), + "SECRET_TOKEN", + ) + .json(&json!({ "name": name, "take_over": take_over })) + .send() + .await?; + Ok::<_, anyhow::Error>((resp.status().as_u16(), resp.text().await?)) + }; + let outcome = async { + let (status, body) = create(plain.clone(), false).await?; + assert_eq!( + status, 409, + "an existing role is not taken without asking: {body}" + ); + + // The group's member would inherit every grant the data table role gets. + let (status, body) = create(group.clone(), true).await?; + assert_eq!(status, 400, "{body}"); + assert!( + body.contains(&member), + "the refusal names the member: {body}" + ); + + let (status, body) = create(plain.clone(), true).await?; + assert_eq!(status, 200, "{body}"); + let catalog = windmill_common::datatable_roles::read_role_catalog( + &db, + windmill_common::datatable_roles::DatatableRoleCluster::Instance, + ) + .await?; + let taken = catalog + .values() + .find(|r| r.name == plain) + .ok_or_else(|| anyhow::anyhow!("{plain} is not in the catalog: {catalog:?}"))?; + let options = (*db.connect_options()) + .clone() + .username(&plain) + .password(taken.pwd.as_deref().unwrap_or_default()); + let mut login = ::connect_with(&options).await?; + sqlx::query("SELECT 1").execute(&mut login).await?; + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide and outlive this test's database, whatever happened above. + for name in [&member, &group, &plain] { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} + #[cfg(all(feature = "private", feature = "enterprise"))] #[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] async fn a_role_delete_that_fails_part_way_leaves_the_role_disabled( @@ -1764,3 +1847,222 @@ async fn cleanup_waits_out_a_save_racing_it_for_an_instance_database( assert_eq!(users, vec!["test-workspace".to_string()]); Ok(()) } + +#[cfg(all(feature = "private", feature = "enterprise"))] +#[sqlx::test(migrations = "../migrations", fixtures("base", "datatable_roles"))] +async fn a_connection_naming_no_role_holds_the_default_one_and_the_viewers( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::workspaces::{get_datatable_resource_from_db, DatatableAccess}; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let suffix: String = uuid::Uuid::new_v4().simple().to_string()[..8].to_string(); + let names = [ + format!("wmtest_floor_{suffix}"), + format!("wmtest_emea_{suffix}"), + format!("wmtest_amer_{suffix}"), + ]; + let roles_url = format!("http://localhost:{port}/api/settings/datatable_roles"); + const MEMBER_LOGINS: &str = "SELECT m.rolname::text FROM pg_auth_members am + JOIN pg_roles r ON r.oid = am.roleid JOIN pg_roles m ON m.oid = am.member + WHERE r.rolname = $1 AND m.rolname LIKE 'wm\\_set\\_%'"; + + // The job is the app publisher's in every case: only the viewer the server recorded differs. + let connect = |role: Option<&'static str>, viewer: Option<&'static str>| { + get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + role, + DatatableAccess::PermissionedAs { + permissioned_as: "u/test-user", + email: "test@windmill.dev", + end_user_email: viewer, + }, + ) + }; + let set_permissions = |default_role: String, roles: Value| { + sqlx::query( + "UPDATE workspace_settings + SET datatable = jsonb_set(datatable, '{datatables,main,permissions}', $1) + WHERE workspace_id = 'test-workspace'", + ) + .bind(json!({ "default_role": default_role, "roles": roles })) + .execute(&db) + }; + + let outcome = async { + let mut ids = vec![]; + for name in &names { + let created: Value = authed(client().post(&roles_url), "SECRET_TOKEN") + .json(&json!({ "name": name })) + .send() + .await? + .error_for_status()? + .json() + .await?; + ids.push(created["id"].as_str().unwrap().to_string()); + } + // `test-user-2` is named by both regional roles, one of them through a group, and by + // `admin`; `test-user-3` by one; the workspace admin `test-user` by neither. + let roles = json!({ + "admin": {"tenants": ["u/test-user-2"]}, + (ids[0].as_str()): {"tenants": ["*"]}, + (ids[1].as_str()): {"tenants": ["u/test-user-2", "u/test-user-3"]}, + (ids[2].as_str()): {"tenants": ["g/analysts"]}, + }); + set_permissions(ids[0].clone(), roles.clone()).await?; + sqlx::query( + "INSERT INTO usr_to_group (workspace_id, usr, group_) + VALUES ('test-workspace', 'test-user-2', 'analysts')", + ) + .execute(&db) + .await?; + let member_of = |login: String| async { + let mut roles: Vec = sqlx::query_scalar( + "SELECT r.rolname::text FROM pg_auth_members am + JOIN pg_roles r ON r.oid = am.roleid JOIN pg_roles m ON m.oid = am.member + WHERE m.rolname = $1", + ) + .bind(login) + .fetch_all(&db) + .await?; + roles.sort(); + Ok::<_, anyhow::Error>(roles) + }; + + let all = connect(None, Some("test2@windmill.dev")).await?; + let login = all["user"].as_str().unwrap().to_string(); + assert!(login.starts_with("wm_set_"), "{login}"); + let mut expected = names.to_vec(); + expected.sort(); + assert_eq!( + member_of(login.clone()).await?, + expected, + "the login holds the default role and the viewer's, and not `admin`" + ); + assert_eq!(connect(None, Some("test2@windmill.dev")).await?, all); + + let two = connect(None, Some("test3@windmill.dev")).await?; + let mut expected = names[..2].to_vec(); + expected.sort(); + assert_eq!( + member_of(two["user"].as_str().unwrap().to_string()).await?, + expected + ); + + // An admin may use every role, the one owning the tables included. As a viewer they add + // what names them, which here is nothing; nor does a job with no viewer recorded. + for viewer in [Some("test@windmill.dev"), None] { + assert_eq!(connect(None, viewer).await?["user"], names[0].as_str()); + } + // A named role is that role alone, whoever is viewing. + let named: &'static str = Box::leak(names[1].clone().into_boxed_str()); + assert_eq!( + connect(Some(named), Some("test2@windmill.dev")).await?["user"], + names[1].as_str() + ); + // A default of `admin` is the connection it always was: the viewer's roles are not added. + set_permissions("admin".to_string(), roles.clone()).await?; + let admin = connect(None, Some("test2@windmill.dev")).await?; + assert_eq!(admin, connect(Some("admin"), None).await?); + assert!(!admin["user"].as_str().unwrap().starts_with("wm_")); + + // With no default role the viewer's roles are all there is, the one open to everyone + // included, and a connection left with none is refused rather than resolved as anything. + set_permissions("$none".to_string(), roles.clone()).await?; + assert_eq!( + connect(None, Some("test3@windmill.dev")).await?, + two, + "no default changes nothing for a viewer the roles name" + ); + assert!(connect(None, None).await.is_err()); + // The same holds for a default the job's own identity is not a tenant of. + let mut closed = roles.clone(); + closed[ids[0].as_str()] = json!({"tenants": []}); + set_permissions(ids[0].clone(), closed).await?; + let as_member = |viewer: Option<&'static str>| { + get_datatable_resource_from_db( + &db, + "test-workspace", + "main", + None, + DatatableAccess::PermissionedAs { + permissioned_as: "u/test-user-3", + email: "test3@windmill.dev", + end_user_email: viewer, + }, + ) + }; + assert_eq!( + as_member(Some("test3@windmill.dev")).await?["user"], + names[1].as_str() + ); + assert!(as_member(None).await.is_err()); + set_permissions(ids[0].clone(), roles).await?; + + // A set login nobody connects with any more is dropped, a recent one is kept, and a + // dropped one is made again for the next viewer who needs it. + let exists = || async { + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = $1)", + ) + .bind(&login) + .fetch_one(&db) + .await + }; + windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await?; + assert!(exists().await?, "a set login in use was swept"); + sqlx::query("UPDATE datatable_role_set SET last_used_at = now() - interval '31 days'") + .execute(&db) + .await?; + windmill_common::datatable_roles::sweep_unused_role_set_logins(&db).await?; + assert!(!exists().await?, "an unused set login was kept"); + assert_eq!( + connect(None, Some("test2@windmill.dev")).await?["user"], + login.as_str() + ); + + // A disabled role must not stay reachable through a login that is a member of it. + authed( + client().post(format!("{roles_url}/{}", ids[2])), + "SECRET_TOKEN", + ) + .json(&json!({ "enabled": false })) + .send() + .await? + .error_for_status()?; + let can_login: bool = + sqlx::query_scalar("SELECT rolcanlogin FROM pg_roles WHERE rolname = $1") + .bind(&login) + .fetch_one(&db) + .await?; + assert!(!can_login, "{login} still logs in with a disabled role"); + assert_eq!( + connect(None, Some("test2@windmill.dev")).await?["user"], + two["user"] + ); + Ok::<_, anyhow::Error>(()) + } + .await; + + // Roles are cluster-wide, so they outlive this test's throwaway database. + for name in &names { + let logins: Vec = sqlx::query_scalar(MEMBER_LOGINS) + .bind(name) + .fetch_all(&db) + .await + .unwrap_or_default(); + for login in logins { + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{login}\"")) + .execute(&db) + .await; + } + let _ = sqlx::query(&format!("DROP ROLE IF EXISTS \"{name}\"")) + .execute(&db) + .await; + } + outcome +} diff --git a/backend/windmill-api-integration-tests/tests/groups.rs b/backend/windmill-api-integration-tests/tests/groups.rs index 26e5caeea1..9928d1c060 100644 --- a/backend/windmill-api-integration-tests/tests/groups.rs +++ b/backend/windmill-api-integration-tests/tests/groups.rs @@ -497,6 +497,72 @@ async fn test_remove_user_from_instance_group_rederives_role( Ok(()) } +/// Clearing an instance group's role, or deleting a group whose role was cleared out from under +/// its members, must take the instance-level role back from them. +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_instance_group_role_revoked_on_clear_and_delete( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let global_base = format!("http://localhost:{port}/api/groups"); + let email = "test2@windmill.dev"; + + let is_super_admin = || async { + sqlx::query_scalar::<_, bool>("SELECT super_admin FROM password WHERE email = $1") + .bind(email) + .fetch_one(&db) + .await + }; + + let resp = authed(client().post(format!("{global_base}/create"))) + .json(&json!({ "name": "admins_grp" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "create: {}", resp.text().await?); + let resp = authed(client().post(format!("{global_base}/update/admins_grp"))) + .json(&json!({ "new_summary": "", "instance_role": "superadmin" })) + .send() + .await?; + assert_eq!(resp.status(), 200, "set role: {}", resp.text().await?); + let resp = authed(client().post(format!("{global_base}/adduser/admins_grp"))) + .json(&json!({ "email": email })) + .send() + .await?; + assert_eq!(resp.status(), 200, "adduser: {}", resp.text().await?); + assert!(is_super_admin().await?, "group should grant superadmin"); + + let resp = authed(client().post(format!("{global_base}/update/admins_grp"))) + .json(&json!({ "new_summary": "", "instance_role": null })) + .send() + .await?; + assert_eq!(resp.status(), 200, "update: {}", resp.text().await?); + assert!( + !is_super_admin().await?, + "clearing the group's role must revoke superadmin" + ); + + // A member left elevated while the group's stored role is already NULL. + sqlx::query( + "UPDATE password SET super_admin = true, role_source = 'instance_group' WHERE email = $1", + ) + .bind(email) + .execute(&db) + .await?; + + let resp = authed(client().delete(format!("{global_base}/delete/admins_grp"))) + .send() + .await?; + assert_eq!(resp.status(), 200, "delete: {}", resp.text().await?); + assert!( + !is_super_admin().await?, + "deleting a role-cleared group must revoke superadmin" + ); + + Ok(()) +} + /// An overwrite import that moves a member from a dropped group to a retained one must keep /// their workspace data. /// diff --git a/backend/windmill-api-integration-tests/tests/jobs_authed.rs b/backend/windmill-api-integration-tests/tests/jobs_authed.rs index c3da2f3fb9..2c5d2ee78e 100644 --- a/backend/windmill-api-integration-tests/tests/jobs_authed.rs +++ b/backend/windmill-api-integration-tests/tests/jobs_authed.rs @@ -702,3 +702,51 @@ async fn test_list_filtered_job_uuids_resolved_excludes_queue( Ok(()) } + +#[sqlx::test(migrations = "../migrations", fixtures("base"))] +async fn test_worker_is_draining_follows_the_job_worker(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let job = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, permissioned_as, kind, tag, args) + VALUES ($1, 'test-workspace', 'test-user', 'u/test-user', 'script', 'deno', '{}'::jsonb)", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, tag, running, started_at, worker) + VALUES ($1, 'test-workspace', now(), 'deno', true, now(), 'wk-drain-test')", + ) + .bind(job) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO worker_ping (worker, worker_instance, ip, worker_group, wm_version) + VALUES ('wk-drain-test', 'host', 'NO IP', 'default', 'v0')", + ) + .execute(&db) + .await?; + + let url = format!("http://localhost:{port}/api/w/test-workspace/jobs/worker_is_draining/{job}"); + let draining = || async { + let resp = authed(client().get(&url)).send().await?; + anyhow::Ok(resp.json::().await?) + }; + assert!(!draining().await?); + + windmill_common::worker::set_worker_draining_query("wk-drain-test", &db).await?; + assert!(draining().await?); + + // A parked job keeps its last worker on the queue row without running on it. + sqlx::query("UPDATE v2_job_queue SET started_at = NULL WHERE id = $1") + .bind(job) + .execute(&db) + .await?; + assert!(!draining().await?); + + Ok(()) +} diff --git a/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs b/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs new file mode 100644 index 0000000000..7033d093f2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs @@ -0,0 +1,570 @@ +use serde_json::json; +use serial_test::serial; +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::invalidate_operator_rights_cache; +use windmill_test_utils::*; + +/// Every test here must be `#[serial]`: the rights cache is process-global and keyed by workspace id +/// alone, so parallel tests under this same id read each other's rights. +const WS: &str = "test-workspace"; + +fn client(token: &str) -> reqwest::Client { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), + ); + reqwest::ClientBuilder::new() + .default_headers(headers) + .build() + .unwrap() +} + +async fn set_builder(db: &Pool, flows: bool, apps: bool) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings SET operator_settings = $1::text::jsonb WHERE workspace_id = $2", + ) + .bind(format!( + r#"{{"builder_flows": {flows}, "builder_apps": {apps}}}"# + )) + .bind(WS) + .execute(db) + .await?; + // The rights are read through a process-global 60s cache keyed by workspace id. + invalidate_operator_rights_cache(WS); + Ok(()) +} + +async fn add_script(db: &Pool, hash: i64, path: &str, owner: &str) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, + summary, description, lock, extra_perms) + VALUES ($1, $2, $3, 'x', 'bun', 'script', $4, '{}', '', '', '', '{}')", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(owner) + .execute(db) + .await?; + Ok(()) +} + +/// A full-code app is deployed multipart, the way the editor and the CLI use it. +fn raw_app_form(app: serde_json::Value) -> reqwest::multipart::Form { + reqwest::multipart::Form::new() + .part( + "app", + reqwest::multipart::Part::text(app.to_string()) + .mime_str("application/json") + .unwrap(), + ) + .part( + "js", + reqwest::multipart::Part::text("console.log(1)").file_name("app.js"), + ) +} + +fn raw_app(path: &str, mode: &str, runnable_path: &str) -> reqwest::multipart::Form { + raw_app_form(json!({ + "path": path, + "summary": "", + "value": {"files": {}, "runnables": {"r": { + "name": "r", "type": "runnableByPath", "runType": "script", + "path": runnable_path + }}}, + "policy": {"execution_mode": mode, "triggerables_v2": {}} + })) +} + +fn composition_flow(path: &str) -> serde_json::Value { + json!({ + "path": path, "summary": "", "description": "", "schema": {}, + "value": {"modules": [{ + "id": "a", + "value": {"type": "script", "path": "u/operator/some_script", "input_transforms": {}} + }]} + }) +} + +/// The app half of the builder boundary, plus the proof that the two rights are independent: a +/// gate reading "either" would authorize the kind its workspace never granted, which is the whole +/// point of splitting them. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_operator_builder_apps_boundary(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let api = format!("http://localhost:{port}/api/w/{WS}"); + let c = client("OPERATOR_TOKEN_1"); + + add_script(&db, 4241, "u/operator/some_script", "operator").await?; + // `permissions_test` gives the operator fixture no rights on `u/alice/**`. + add_script(&db, 4243, "u/alice/private", "alice").await?; + // A version shared with the operator, then archived, and a newer one at the same path that is + // not: the newer one is what runs. + add_script(&db, 4244, "u/alice/reused", "alice").await?; + sqlx::query( + "UPDATE script SET archived = true, extra_perms = '{\"u/operator\": false}', + created_at = now() - interval '1 day' WHERE hash = 4244", + ) + .execute(&db) + .await?; + add_script(&db, 4245, "u/alice/reused", "alice").await?; + + set_builder(&db, false, false).await?; + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/a0", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + !resp.status().is_success(), + "an operator without the builder right must not create a full-code app" + ); + + set_builder(&db, true, true).await?; + + // `*_raw_source` compiles caller-supplied sources with a bundler job on a worker. It sits + // beside `create_app_raw`, which builders MAY use, so it is the gate most likely to be opened + // by mistake later. + let resp = c + .post(format!("{api}/apps/create_raw_source")) + .json(&json!({ + "path": "u/operator/a1", "summary": "", + "value": {"files": {"index.ts": "console.log(1)"}, "runnables": {}}, + "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not compile app sources on a worker" + ); + + // Low-code apps carry inline scripts, so they stay shut too. + let resp = c + .post(format!("{api}/apps/create")) + .json(&json!({ + "path": "u/operator/a2", "summary": "", + "value": {"grid": []}, "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not create a low-code app" + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app("u/operator/a3", "publisher", "u/alice/private")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not deploy an app referencing a runnable it cannot read" + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app("u/operator/a4", "viewer", "u/operator/some_script")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not deploy a viewer-mode app: the policy stops bounding what it can invoke" + ); + + // Storage strips a NUL, turning this key into `inlineScript`. + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app_form(json!({ + "path": "u/operator/a_nul", "summary": "", + "value": {"files": {}, "runnables": {"r": { + "type": "inline", + "inline\u{0}Script": {"content": "x", "language": "bun"} + }}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": {}} + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "a NUL in a key must not hide an inline script: {}", + resp.text().await? + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/a5", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + resp.status().is_success(), + "a builder must be able to deploy a full-code app over readable runnables: {}", + resp.text().await? + ); + let sandbox: Option = sqlx::query_scalar( + "SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2", + ) + .bind(WS) + .bind("u/operator/a5") + .fetch_one(&db) + .await?; + assert_eq!( + sandbox, + Some(true), + "a builder-authored app must be stored sandboxed" + ); + + // `execute_component` looks up `:` with an unrestricted component string, so + // a key with a second colon still resolves at run time and must not slip past validation. + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app_form(json!({ + "path": "u/operator/a6", "summary": "", + "value": {"files": {}, "runnables": {}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": { + "x:y:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}} + }} + }))) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a multi-colon triggerable key must not hide an unreadable runnable from validation" + ); + + // The two rights are independent. + set_builder(&db, true, false).await?; + let resp = c + .post(format!("{api}/flows/create")) + .json(&composition_flow("u/operator/flows_only")) + .send() + .await?; + assert!( + resp.status().is_success(), + "flows-only must still create a flow: {}", + resp.text().await? + ); + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/flows_only_app", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + !resp.status().is_success(), + "flows-only must not deploy a full-code app" + ); + + set_builder(&db, false, true).await?; + let resp = c + .post(format!("{api}/flows/create")) + .json(&composition_flow("u/operator/apps_only")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "apps-only must not create a flow" + ); + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/apps_only_app", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + resp.status().is_success(), + "apps-only must still deploy a full-code app: {}", + resp.text().await? + ); + + // The deploy panel saves access and frontend scopes through the settings-only endpoint. + for (body, expected) in [ + ( + json!({"policy": {"execution_mode": "anonymous", "triggerables_v2": {}}}), + 200, + ), + ( + json!({"policy": {"execution_mode": "publisher", "sandbox": false, "triggerables_v2": {}}}), + 403, + ), + ( + json!({"value": {"files": {}, "runnables": {}}, "policy": {"execution_mode": "publisher"}}), + 403, + ), + ] { + let resp = c + .post(format!("{api}/apps/update/u/operator/apps_only_app")) + .json(&body) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "settings update {body}: {}", + resp.text().await? + ); + } + let sandbox: Option = sqlx::query_scalar( + "SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2", + ) + .bind(WS) + .bind("u/operator/apps_only_app") + .fetch_one(&db) + .await?; + assert_eq!( + sandbox, + Some(true), + "a settings update must not turn a builder app's sandbox off" + ); + + // The editor runs the runnables a draft names as whoever opens it, and its frontend code + // with their session unless the preview is sandboxed. + for (runnables, policy, expected) in [ + (json!({}), json!({"sandbox": true}), 200), + ( + json!({"a": {"type": "inline", "inlineScript": {"content": "x", "language": "bun"}}}), + json!({"sandbox": true}), + 403, + ), + (json!({}), json!({"sandbox": false}), 403), + ( + json!({}), + json!({"sandbox": true, "execution_mode": "viewer"}), + 403, + ), + (json!({}), json!(null), 403), + ( + json!({}), + json!({"sandbox": true, "on_behalf_of": "u/alice", "on_behalf_of_email": "alice@windmill.dev"}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "on_behalf_of": "u/operator"}), + 200, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": { + format!("{}script/u/operator/some_script", "a:".repeat(300)): {} + }}), + 400, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/operator/some_script"}}), + json!({"sandbox": true}), + 200, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/private"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/reused"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "path", "runType": "hubscript", "path": "hub/1/x/y"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "path", "runType": "script/u/alice", "path": "u/operator/some_script"}}), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": "path", "runType": "script", "path": ["u/alice/private"]}}), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": ["path"], "runType": "script", "path": "u/alice/private"}}), + json!({"sandbox": true}), + 400, + ), + ( + json!([{"type": "path", "runType": "script", "path": "u/alice/private"}]), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": "path", "runType": "script", "path": "u/operator/some_script", + "fields": {"x": {"type": "static", "value": {"type": "path", "path": "/srv/out"}}}}}), + json!({"sandbox": true}), + 200, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": {"x:rawscript/abc": {}}}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": {"x:script/u/alice/private": {}}}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables_v2": { + "x:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}}, + "y:rawscript/abc": {"static_inputs": {}, "one_of_inputs": {}} + }}), + 200, + ), + ] { + let resp = c + .post(format!("{api}/drafts/update/raw_app/u/operator/d1")) + .json(&json!({ + "value": {"files": {}, "runnables": runnables, "summary": "", "policy": policy}, + "force": true + })) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "raw app draft {runnables} {policy}: {}", + resp.text().await? + ); + } + + // The component preview runs a deployed runnable for a builder; inline code and hub + // scripts are unreviewed code. + for (preview, expected) in [ + (json!({"path": "script/u/operator/some_script"}), 200), + (json!({"path": "script/hub/1/x/y"}), 403), + ( + json!({"raw_code": {"content": "x", "language": "bun"}}), + 403, + ), + ] { + let mut body = json!({"component": "c", "args": {}, "force_viewer_static_fields": {}}); + body.as_object_mut() + .unwrap() + .extend(preview.as_object().unwrap().clone()); + let resp = c + .post(format!( + "{api}/apps_u/execute_component/u/operator/apps_only_app" + )) + .json(&body) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "component preview {preview}: {}", + resp.text().await? + ); + } + + // Delete and redeploy each serve both kinds; builder rights cover only full-code apps. + let resp = client("SECRET_TOKEN") + .post(format!("{api}/apps/create")) + .json(&json!({ + "path": "u/operator/low_code", "summary": "", + "value": {"grid": []}, "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let resp = c + .post(format!("{api}/apps/update_raw/u/operator/low_code")) + .multipart(raw_app_form(json!({ + "policy": {"execution_mode": "publisher", "triggerables_v2": {}} + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "policy-only update of a low-code app: {}", + resp.text().await? + ); + // Deployment history restores a full-code version with allow_kind_change set. + let resp = c + .post(format!("{api}/apps/update_raw/u/operator/apps_only_app")) + .multipart(raw_app_form(json!({ + "value": {"files": {}, "runnables": {}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": {}}, + "allow_kind_change": true + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "restore of a full-code version: {}", + resp.text().await? + ); + for (path, expected) in [("low_code", 403), ("apps_only_app", 200)] { + let resp = c + .delete(format!("{api}/apps/delete/u/operator/{path}")) + .send() + .await?; + let status = resp.status(); + assert_eq!(status, expected, "delete {path}: {}", resp.text().await?); + } + + Ok(()) +} + +/// The readability check must not hold its RLS transaction while it takes a second connection, +/// or a one-connection pool (`DATABASE_CONNECTIONS=1`) stalls every builder draft naming a script. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_operator_builder_app_check_fits_one_connection( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) -> anyhow::Result<()> { + initialize_tracing().await; + let db = pool_opts + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(5)) + .connect_with(connect_opts) + .await?; + let server = ApiServer::start(db.clone()).await?; + let api = format!("http://localhost:{}/api/w/{WS}", server.addr.port()); + add_script(&db, 4241, "u/operator/some_script", "operator").await?; + set_builder(&db, false, true).await?; + + let resp = client("OPERATOR_TOKEN_1") + .post(format!("{api}/drafts/update/raw_app/u/operator/d1")) + .json(&json!({ + "value": {"files": {}, "summary": "", "runnables": {"r": { + "name": "r", "type": "runnableByPath", "runType": "script", + "path": "u/operator/some_script" + }}, "policy": {"sandbox": true, "triggerables_v2": {}}}, + "force": true + })) + .send() + .await?; + let status = resp.status(); + assert_eq!(status, 200, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index 5e08949138..c6c2a1fede 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -33,6 +33,7 @@ use windmill_common::{ flow_conversations::{ add_message_to_conversation_tx, message_attachments, MessageExtras, MessageType, }, + flows::Retry, get_latest_flow_version_info_for_path, jobs::{ check_tag_available_for_workspace_internal, format_result, script_path_to_payload, @@ -126,6 +127,71 @@ pub async fn drop_unclaimable_run_lineage( Ok(()) } +/// Applies the run's `retry` policy by handing `push` a one-step-flow request, which it +/// materializes as a native retryable `Script`. +pub fn with_run_retry( + run_query: &RunJobQuery, + payload: JobPayload, + args: &PushArgs<'_>, + tag: &Option, +) -> error::Result { + let Some(retry) = run_query.retry.as_deref() else { + return Ok(payload); + }; + // Retry attempts are always pushed visible to the owner, which would leak a private run. + if run_query.invisible_to_owner.unwrap_or(false) { + return Err(Error::BadRequest( + "retry cannot be combined with invisible_to_owner".to_string(), + )); + } + let retry: Retry = serde_json::from_str(retry) + .map_err(|e| Error::BadRequest(format!("invalid retry policy: {e}")))?; + let JobPayload::ScriptHash { + hash, + path, + cache_ttl, + cache_ignore_s3_path, + language, + priority, + apply_preprocessor, + debouncing_settings, + concurrency_settings, + .. + } = payload + else { + return Err(Error::BadRequest( + "retry is only supported for workspace scripts".to_string(), + )); + }; + // A failed attempt keeps its raw args and the retry re-push skips the preprocessor, so + // every retry would call `main` with un-preprocessed input. + if apply_preprocessor { + return Err(Error::BadRequest( + "retry is not supported for a script with a preprocessor unless skip_preprocessor is set" + .to_string(), + )); + } + Ok(JobPayload::SingleStepFlow { + path, + hash: Some(hash), + flow_version: None, + language: Some(language), + args: HashMap::from(args), + retry: Some(retry), + error_handler_path: None, + error_handler_args: None, + skip_handler: None, + cache_ttl, + cache_ignore_s3_path, + priority, + tag_override: tag.clone(), + trigger_path: None, + apply_preprocessor, + concurrency_settings, + debouncing_settings, + }) +} + /// The jobs of `referenced` that `authed` cannot claim as its own run lineage: anything but the /// token's own job and that job's `parent_job`, `root_job` and `flow_innermost_root_job`, or for /// a workspace admin anything outside the workspace. A restricted job token never gets the admin @@ -179,6 +245,27 @@ pub async fn unclaimable_run_lineage( Ok(referenced) } +/// `WM_END_USER_EMAIL` of a run started through a `/jobs/run*` route. `WM_EMAIL` names the +/// runnable's `on_behalf_of` identity when it has one, so this is the only place the caller's +/// address reaches the job. A job's own `WM_TOKEN` passes on the end user of that job, and a run +/// a trigger fired has no end user. +pub async fn run_end_user_email( + db: &DB, + w_id: &str, + authed: &ApiAuthed, + trigger: Option<&TriggerMetadata>, +) -> error::Result> { + if trigger.is_some() { + return Ok(None); + } + match authed.job_id { + Some(job_id) => Ok(windmill_common::auth::get_job_perms(db, &job_id, w_id) + .await? + .and_then(|p| p.end_user_email)), + None => Ok(Some(authed.email.clone())), + } +} + #[cfg(feature = "enterprise")] pub async fn check_license_key_valid() -> error::Result<()> { use windmill_common::ee_oss::LICENSE_KEY_VALID; @@ -947,6 +1034,7 @@ pub async fn run_flow<'c>( ) }; + let end_user_email = run_end_user_email(db, w_id, authed, trigger.as_ref()).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, mut tx) = push( &db, @@ -982,7 +1070,7 @@ pub async fn run_flow<'c>( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -1151,6 +1239,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( let tag = run_query.tag.clone().or(tag); let push_args = PushArgs { args: &args.args, extra: args.extra }; check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; let return_tx = tx_o.is_some(); @@ -1177,6 +1266,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, trigger.as_ref()).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -1210,7 +1300,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( }, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, scope_ceiling.as_deref(), diff --git a/backend/windmill-api-jobs/src/types.rs b/backend/windmill-api-jobs/src/types.rs index f37040bd11..f50cb61451 100644 --- a/backend/windmill-api-jobs/src/types.rs +++ b/backend/windmill-api-jobs/src/types.rs @@ -52,6 +52,8 @@ pub struct RunJobQuery { pub trigger_external_id: Option, pub service_name: Option, pub suspended_mode: Option, + /// A JSON-encoded `Retry` policy for a script run; see `with_run_retry`. + pub retry: Option, } impl RunJobQuery { diff --git a/backend/windmill-api-schedule/src/lib.rs b/backend/windmill-api-schedule/src/lib.rs index 855f687431..b8ee9ded91 100644 --- a/backend/windmill-api-schedule/src/lib.rs +++ b/backend/windmill-api-schedule/src/lib.rs @@ -1492,7 +1492,8 @@ async fn set_default_error_handler( } else { windmill_api_settings::delete_global_setting(&db, key.as_str()).await?; } - windmill_api_settings::audit_setting_write(&db, &authed, &key, value.as_ref()).await?; + windmill_audit::instance_config::audit_setting_write(&db, &authed, &key, value.as_ref(), None) + .await?; if payload.override_existing { // The rewrite and its history rows go in one transaction: on separate diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index f622c4bb91..4f4c962759 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -4266,12 +4266,6 @@ async fn delete_script_by_path( check_scopes(&authed, || format!("scripts:write:{}", path))?; - if path == "u/admin/hub_sync" && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -4445,12 +4439,6 @@ async fn delete_scripts_bulk( require_admin(authed.is_admin, &authed.username)?; - if request.paths.contains(&"u/admin/hub_sync".to_string()) && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index c79b596617..1359e8c142 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -29,7 +29,12 @@ mod log_cleanup; mod storage_usage; use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed}; -use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_audit::{ + audit_oss::audit_log, + instance_config::{audit_setting_write, InstanceConfigAuditor}, + ActionKind, +}; +use windmill_common::hub_resource_types::{self, SyncSource}; use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT; use windmill_common::DB; @@ -58,10 +63,10 @@ use windmill_common::{ error::{self, pg_error_message, JsonResult, Result}, get_database_url, global_settings::{ - ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, - AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXTERNAL_INSTANCE_PG_SETTING, + ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING, + APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -832,50 +837,7 @@ pub async fn set_global_setting( require_super_admin(&db, &authed).await?; let value = value.value.unwrap_or(serde_json::Value::Null); set_global_setting_internal(&db, key.clone(), value.clone()).await?; - audit_setting_write(&db, &authed, &key, Some(&value)).await -} - -/// A setting is recorded by name, with its value only when that is a boolean or a number. -/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no -/// list of secret-bearing settings stays complete as settings are added. -/// -/// AUTHORIZATION: records a write, checks nothing. The caller must have established that -/// `authed` may change instance settings before performing the write this records. -pub async fn audit_setting_write( - db: &DB, - authed: &ApiAuthed, - key: &str, - value: Option<&serde_json::Value>, -) -> error::Result<()> { - let deleted = match value { - None | Some(serde_json::Value::Null) => true, - Some(serde_json::Value::String(s)) => s.trim().is_empty(), - Some(_) => false, - }; - let recorded = match value { - Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => { - Some(v.to_string()) - } - _ => None, - }; - audit_log( - db, - authed, - if deleted { - "global_settings.delete" - } else { - "global_settings.update" - }, - if deleted { - ActionKind::Delete - } else { - ActionKind::Update - }, - "global", - Some(key), - recorded.as_deref().map(|v| [("value", v)].into()), - ) - .await + audit_setting_write(&db, &authed, &key, Some(&value), None).await } async fn audit_instance_action( @@ -1238,26 +1200,22 @@ async fn run_setting_pre_write_hook( } } } - GITHUB_APP_WEBHOOK_BASE_URL_SETTING => { - // A bad value here yields a webhook GitHub can never deliver to, and the + GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => { + // A bad webhook base yields a webhook GitHub can never deliver to, and the // failure only shows up much later as "falling back to polling" on a - // repository — so reject it at the boundary instead. + // repository; a bad API base puts an unreachable URL in front of every user. + // So reject both at the boundary instead. match value { // Clearing (delete row) is handled by the caller; allow it through. serde_json::Value::Null => {} serde_json::Value::String(s) if s.trim().is_empty() => {} serde_json::Value::String(s) => { - windmill_common::global_settings::validate_webhook_base_url(s).map_err( - |e| { - error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}" - )) - }, - )?; + windmill_common::global_settings::validate_webhook_base_url(s) + .map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?; } _ => { return Err(error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string" + "{key} must be a URL string" ))); } } @@ -1370,61 +1328,17 @@ async fn set_instance_config( run_setting_pre_write_hook(&db, key, value).await?; } } + let auditor = InstanceConfigAuditor { db: &db, author: &authed, source: None }; // The external-instance write removes its own key from the diff. - let external = settings_diff - .upserts - .get(EXTERNAL_INSTANCE_PG_SETTING) - .cloned() - .map(Some) - .or_else(|| { - settings_diff - .deletes - .iter() - .any(|k| k == EXTERNAL_INSTANCE_PG_SETTING) - .then_some(None) - }); windmill_common::external_instance_pg::write_external_instance_pg_from_diff( &db, &mut settings_diff, + &auditor, ) .await?; - if let Some(value) = external { - audit_setting_write(&db, &authed, EXTERNAL_INSTANCE_PG_SETTING, value.as_ref()).await?; - } - - // Applied and audited one setting at a time: the writes are not transactional, so a - // batch that fails midway must not leave the settings it did write unrecorded. - let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } = - settings_diff; - let steps = upserts - .into_iter() - .map(|(k, v)| (k, Some(v))) - .chain(deletes.into_iter().map(|k| (k, None))) - .map(Some) - .chain(std::iter::once(None)); - for step in steps { - let mut diff = instance_config::SettingsDiff::default(); - match &step { - Some((key, value)) => { - if let Some(previous) = previous_values.remove(key) { - diff.previous_values.insert(key.clone(), previous); - } - match value { - Some(value) => { - diff.upserts.insert(key.clone(), value.clone()); - } - None => diff.deletes.push(key.clone()), - } - } - None => diff.unchanged_count = unchanged_count, - } - instance_config::apply_settings_diff(&db, &diff) - .await - .map_err(|e| error::Error::internal_err(e.to_string()))?; - if let Some((key, value)) = &step { - audit_setting_write(&db, &authed, key, value.as_ref()).await?; - } - } + instance_config::apply_settings_diff(&db, &settings_diff, &auditor) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; if ai_config_changed { bump_instance_ai_config_revision(); @@ -1454,37 +1368,13 @@ async fn set_instance_config( .collect(); let configs_diff = instance_config::diff_worker_configs(¤t_wc, &desired_wc, ApplyMode::Merge); - let steps = configs_diff - .upserts - .into_iter() - .map(|(g, c)| (g, Some(c))) - .chain(configs_diff.deletes.into_iter().map(|g| (g, None))); - for (group, config) in steps { - let mut diff = instance_config::ConfigsDiff::default(); - let (operation, action_kind) = match config { - Some(config) => { - diff.upserts.insert(group.clone(), config); - ("worker_config.update", ActionKind::Update) - } - None => { - diff.deletes.push(group.clone()); - ("worker_config.delete", ActionKind::Delete) - } - }; - instance_config::apply_configs_diff(&db, &diff) - .await - .map_err(|e| error::Error::internal_err(e.to_string()))?; - audit_log( - &db, - &authed, - operation, - action_kind, - "global", - Some(&format!("worker__{group}")), - None, - ) - .await?; - } + instance_config::apply_configs_diff( + &db, + &configs_diff, + &InstanceConfigAuditor { db: &db, author: &authed, source: None }, + ) + .await + .map_err(|e| error::Error::internal_err(e.to_string()))?; } Ok(()) @@ -2576,95 +2466,6 @@ pub async fn get_jwks() -> JsonResult { Ok(Json(JwksResponse { keys: vec![] })) } -#[derive(serde::Deserialize, serde::Serialize)] -struct CachedResourceType { - #[allow(dead_code)] - id: i64, - name: String, - schema: Option, - #[allow(dead_code)] - app: String, - description: Option, - /// Doubly optional, and read through a wrapping deserializer: this struct also - /// decodes the on-disk cache, where an absent key means "written before the - /// column, leave the stored extension alone" and an explicit null means the hub - /// dropped it. Plain serde folds both into `None`. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - format_extension: Option>, - /// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit - /// null (the hub naming nothing) clears it. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - display_name: Option>, -} - -#[derive(serde::Deserialize)] -struct HubResourceTypeRaw { - id: i64, - name: String, - schema: Option, - app: String, - description: Option, - #[serde(default)] - format_extension: Option, - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - display_name: Option>, -} - -async fn fetch_resource_types_from_hub() -> error::Result> { - let response = HTTP_CLIENT - .get(format!( - "{}/resource_types/list", - windmill_common::DEFAULT_HUB_BASE_URL - )) - .header("Accept", "application/json") - .send() - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to fetch from hub: {}", e)))?; - - if !response.status().is_success() { - return Err(error::Error::InternalErr(format!( - "Hub returned status {}", - response.status() - ))); - } - - let raw_types: Vec = response - .json() - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to parse hub response: {}", e)))?; - - Ok(raw_types - .into_iter() - .filter_map(|rt| { - let schema = match rt.schema { - Some(s) => match serde_json::from_str(&s) { - Ok(v) => Some(v), - Err(_) => return None, - }, - None => None, - }; - Some(CachedResourceType { - id: rt.id, - name: rt.name, - schema, - app: rt.app, - description: rt.description, - format_extension: Some(rt.format_extension), - display_name: rt.display_name, - }) - }) - .collect()) -} - #[derive(serde::Deserialize)] struct SyncResourceTypesQuery { name: Option, @@ -2677,118 +2478,45 @@ async fn sync_cached_resource_types( ) -> error::Result { require_super_admin(&db, &authed).await?; - use windmill_common::worker::HUB_RT_CACHE_DIR; - let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); + let outcome = hub_resource_types::sync(&db).await?; - // Manual sync is hub-first so it lands newly-published hub types on demand. The - // on-disk cache is only a fallback for when the hub is unreachable (airgapped - // installs / network error); refreshing it is left to the daily cache-rt cron and - // the startup sync in main.rs, which own the offline path. - let (resource_types, from_hub) = match fetch_resource_types_from_hub().await { - Ok(types) => { - tracing::info!("Fetched {} resource types live from the hub", types.len()); - (types, true) - } - Err(hub_err) => { - tracing::warn!( - "Live hub fetch failed ({hub_err}), falling back to on-disk cache at {cache_path}" - ); - match tokio::fs::read_to_string(&cache_path).await { - Ok(content) => { - let parsed = serde_json::from_str::>(&content) - .map_err(|e| { - error::Error::InternalErr(format!( - "Failed to parse cached resource types: {}", - e - )) - })?; - (parsed, false) - } - Err(_) => return Err(hub_err), - } - } - }; - - let mut synced_count = 0; - - for rt in &resource_types { - // A name too long for the column counts as absent, leaving the stored one alone: one bad - // entry must not fail the upsert and end the rest of the sync. - let display_name = match &rt.display_name { - Some(Some(name)) if name.chars().count() > 100 => None, - other => other.clone(), + if !outcome.changed.is_empty() { + let source = match outcome.source { + SyncSource::Hub => "hub", + SyncSource::ImageCache => "image_cache", }; - let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", - &rt.name, - rt.schema.as_ref(), - rt.description.as_deref(), - rt.format_extension.clone().flatten(), - rt.format_extension.is_some(), - display_name.clone().flatten(), - display_name.is_some(), + let changed = outcome.changed.join(", "); + audit_log( + &db, + &authed, + "resource_types.sync", + ActionKind::Update, + "admins", + None, + Some([("source", source), ("changed", changed.as_str())].into()), ) - .fetch_one(&db) .await?; - - if exists.unwrap_or(false) { - continue; - } - - sqlx::query!( - // Whether the payload carried the key at all is what decides: present - // (even as null) is authoritative and may clear, absent means a cache - // written before the column and must leave the stored value alone. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) - VALUES ('admins', $1, $2, $3, $4, $6, now()) - ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, - -- A fileset is a set of files, so it cannot also be one file. - -- Create and update reject the pair; this writer bypasses both, so - -- it declines the extension rather than persisting the forbidden - -- combination onto a same-named local fileset. - format_extension = CASE - WHEN resource_type.is_fileset THEN NULL - WHEN $5 THEN EXCLUDED.format_extension - ELSE resource_type.format_extension END, - display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END, - edited_at = now()", - &rt.name, - rt.schema.as_ref(), - rt.description.as_deref(), - rt.format_extension.clone().flatten(), - rt.format_extension.is_some(), - display_name.clone().flatten(), - display_name.is_some(), - ) - .execute(&db) - .await?; - - synced_count += 1; } // If a specific type was requested and is still absent after syncing, surface an // explicit not-found instead of a silent "Synced 0". Word it by source so the // cache-fallback path does not claim it checked the hub. if let Some(name) = name.as_deref() { - if !resource_types.iter().any(|rt| rt.name == name) { - let source = if from_hub { - "on the hub" - } else { - "in the cached resource types (hub unreachable)" + if !outcome.listed.iter().any(|listed| listed == name) { + let source = match outcome.source { + SyncSource::Hub => "on the hub", + SyncSource::ImageCache => "in the cached resource types (hub unavailable)", }; return Err(error::Error::NotFound(format!( - "resource type '{}' not found {}", - name, source + "resource type '{name}' not found {source}" ))); } } - Ok(format!( - "Synced {} resource types ({} unchanged)", - synced_count, - resource_types.len() - synced_count - )) + Ok(match outcome.source { + SyncSource::Hub => outcome.summary(), + SyncSource::ImageCache => format!("{}, the hub being unavailable", outcome.summary()), + }) } #[cfg(test)] diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index ac9defcd53..46bd6825e8 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -490,7 +490,7 @@ async fn list_user_usage( " SELECT usr.email, usage.executions FROM usr, LATERAL ( - SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index 0102f1b033..8ebee2a967 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -41,6 +41,7 @@ pub fn global_service() -> Router { .route("/queue_metrics", get(get_queue_metrics)) .route("/queue_metrics_series", get(get_queue_metrics_series)) .route("/queue_status", get(get_queue_status)) + .route("/runnable_stats", get(get_runnable_stats)) .route("/queue_counts", get(get_queue_counts)) .route("/queue_running_counts", get(get_queue_running_counts)) .route( @@ -86,6 +87,7 @@ struct WorkerPing { job_isolation: Option, #[serde(skip_serializing_if = "Option::is_none")] native_mode: Option, + draining: bool, } // #[derive(Serialize, Deserialize)] @@ -118,7 +120,7 @@ async fn list_worker_pings( WorkerPing, "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, - custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation, native_mode, draining FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", @@ -298,6 +300,81 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } +#[derive(Deserialize)] +struct RunnableStatsQuery { + window_secs: Option, + worker_group: Option, + workspace: Option, + order_by: Option, + limit: Option, +} + +#[derive(Serialize)] +struct RunnableStats { + workspace_id: String, + runnable_path: String, + worker_group: String, + job_count: i64, + total_duration_ms: i64, + max_memory_peak: i32, + sum_memory_peak: i64, + memory_sample_count: i64, + total_cpu_ms: i64, +} + +async fn get_runnable_stats( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult> { + require_devops_role(&db, &authed).await?; + + let window = query.window_secs.unwrap_or(24 * 3600).clamp( + 3600, + windmill_common::runnable_job_stats::RETENTION_DAYS * 24 * 3600, + ); + // Rows are hourly buckets keyed by their start, so the bucket the window begins in + // is included whole. + let from_hour = (chrono::Utc::now().timestamp() - window) / 3600 * 3600; + let order_by = match query.order_by.as_deref() { + Some("max_memory") => "max_memory", + Some("cpu") => "cpu", + _ => "total_duration", + }; + + let rows = sqlx::query_as!( + RunnableStats, + r#" + SELECT workspace_id, runnable_path, worker_group, + SUM(job_count)::bigint AS "job_count!", + SUM(total_duration_ms)::bigint AS "total_duration_ms!", + MAX(max_memory_peak) AS "max_memory_peak!", + SUM(sum_memory_peak)::bigint AS "sum_memory_peak!", + SUM(memory_sample_count)::bigint AS "memory_sample_count!", + SUM(total_cpu_ms)::bigint AS "total_cpu_ms!" + FROM runnable_job_stats + WHERE hour >= $1 + AND ($2::text IS NULL OR worker_group = $2) + AND ($3::text IS NULL OR workspace_id = $3) + GROUP BY workspace_id, runnable_path, worker_group + ORDER BY CASE $4::text + WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric + WHEN 'cpu' THEN SUM(total_cpu_ms) + ELSE SUM(total_duration_ms) + END DESC, workspace_id, runnable_path, worker_group + LIMIT $5 + "#, + from_hour, + query.worker_group, + query.workspace, + order_by, + query.limit.unwrap_or(100).clamp(1, 1000), + ) + .fetch_all(&db) + .await?; + Ok(Json(rows)) +} + #[derive(Deserialize)] struct QueueMetricsSeriesQuery { window_secs: Option, diff --git a/backend/windmill-api-workspaces/src/datatable_acl.rs b/backend/windmill-api-workspaces/src/datatable_acl.rs index ebab6214ce..efaa712481 100644 --- a/backend/windmill-api-workspaces/src/datatable_acl.rs +++ b/backend/windmill-api-workspaces/src/datatable_acl.rs @@ -683,6 +683,9 @@ pub(crate) struct CatalogFacts { pub(crate) former_owner: Option, /// For a revoke: each grant it takes back. pub(crate) revoked_grants: Vec, + /// For a grant on what a schema holds: the role cannot reach the schema yet, so the grant + /// would give it nothing it can use. + pub(crate) lacks_schema_usage: bool, } /// A grant a revoke takes back, as the catalog records it: what `source` gave on `object` (the @@ -1526,7 +1529,24 @@ async fn build_plan( ))); } } - let facts = CatalogFacts { other_pg_roles, existing_objects, former_owner, revoked_grants }; + let lacks_schema_usage = match (&change, target) { + (AclChange::Grant { scope, .. }, AclTarget::Schema { schema }) + if !matches!(scope, GrantScope::Target) => + { + !has_schema_usage(client, schema, &pg_role).await? + } + (AclChange::Grant { .. }, AclTarget::Table { schema, .. }) => { + !has_schema_usage(client, schema, &pg_role).await? + } + _ => false, + }; + let facts = CatalogFacts { + other_pg_roles, + existing_objects, + former_owner, + revoked_grants, + lacks_schema_usage, + }; let mut plan = crate::datatable_acl_oss::plan_statements(target, &change, dbname, &pg_role, &facts)?; if matches!(change, AclChange::SetOwner { .. }) { @@ -1540,6 +1560,27 @@ async fn build_plan( Ok(plan) } +/// Whether `pg_role` may look into `schema`, directly or through a role it inherits from. +async fn has_schema_usage( + client: &tokio_postgres::Client, + schema: &str, + pg_role: &str, +) -> Result { + Ok(client + .query_one( + "SELECT has_schema_privilege($1::name, $2::text, 'USAGE')", + &[&pg_role, &schema], + ) + .await + .map_err(|e| { + Error::internal_err(format!( + "Failed to read the role's privileges on schema {schema}: {}", + pg_error_message(&e) + )) + })? + .get(0)) +} + /// Postgres only hands an object to a role that could have created it: a table to one with /// `CREATE` on its schema, a schema to one with `CREATE` on the database. async fn missing_owner_privilege( diff --git a/backend/windmill-api-workspaces/src/datatable_migrations.rs b/backend/windmill-api-workspaces/src/datatable_migrations.rs index e63e69d4d5..21e460359e 100644 --- a/backend/windmill-api-workspaces/src/datatable_migrations.rs +++ b/backend/windmill-api-workspaces/src/datatable_migrations.rs @@ -880,7 +880,11 @@ struct DatatableMigrationsStatusResult { /// Whether the SQL-migrations feature is enabled for a data table. Honors the /// explicit `migrations_enabled` flag; when unset (data tables predating the /// feature) it is considered enabled only if migrations already exist. -async fn datatable_migrations_enabled(db: &DB, w_id: &str, datatable_name: &str) -> Result { +pub(crate) async fn datatable_migrations_enabled( + db: &DB, + w_id: &str, + datatable_name: &str, +) -> Result { let flag: Option = sqlx::query_scalar!( "SELECT (ws.datatable->'datatables'->$2->>'migrations_enabled')::boolean \ FROM workspace_settings ws WHERE ws.workspace_id = $1", diff --git a/backend/windmill-api-workspaces/src/remote_deploy.rs b/backend/windmill-api-workspaces/src/remote_deploy.rs index 3e701cbf14..a0f922a607 100644 --- a/backend/windmill-api-workspaces/src/remote_deploy.rs +++ b/backend/windmill-api-workspaces/src/remote_deploy.rs @@ -25,11 +25,11 @@ use axum::{ }; use chrono::{DateTime, Utc}; use lazy_static::lazy_static; -use magic_crypt::MagicCrypt256; use regex::Regex; use serde::{Deserialize, Serialize}; use windmill_api_auth::{is_effectively_unscoped, ApiAuthed, Tokened}; use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::variables::WorkspaceCrypt; use windmill_common::{ auth::hash_token, error::{error_source_chain, Error, Result}, @@ -676,8 +676,8 @@ async fn proxy( pub(crate) async fn reencrypt_tokens( conn: &mut sqlx::PgConnection, w_id: &str, - old: &MagicCrypt256, - new: &MagicCrypt256, + old: &WorkspaceCrypt, + new: &WorkspaceCrypt, ) -> Result<()> { let rows = sqlx::query!( "SELECT email, token FROM remote_deploy_token diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index d5bc5c549b..44a7c975b4 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -2326,6 +2326,9 @@ struct DataTableTables { /// What the role the listing connected as may create. can_create_schema: bool, creatable_schemas: Vec, + /// Whether its schema is kept as versioned migrations, which a change made with plain SQL + /// would bypass. + migrations_enabled: bool, } #[derive(Deserialize)] @@ -2571,8 +2574,17 @@ async fn list_one_datatable_tables( default_role: windmill_common::datatable_roles::ADMIN_DATATABLE_ROLE.to_string(), can_create_schema: false, creatable_schemas: vec![], + migrations_enabled: false, }; let result: Result<()> = async { + // Read from the workspace settings alone, so it is known even when the database is not + // reachable. + entry.migrations_enabled = crate::datatable_migrations::datatable_migrations_enabled( + db, + w_id, + &entry.datatable_name, + ) + .await?; let governing = resolve_governing_datatable(db, w_id, &entry.datatable_name).await?; entry.instance = governing.role_cluster().is_some(); let usable = @@ -6318,8 +6330,8 @@ async fn set_encryption_key( async fn reencrypt_git_sync_secrets( conn: &mut sqlx::PgConnection, w_id: &str, - old: &magic_crypt::MagicCrypt256, - new: &magic_crypt::MagicCrypt256, + old: &windmill_common::variables::WorkspaceCrypt, + new: &windmill_common::variables::WorkspaceCrypt, ) -> Result<()> { let Some((mut credentials, mut git_sync)) = sqlx::query_as::<_, (serde_json::Value, Option)>( @@ -11608,10 +11620,13 @@ struct ChangeOperatorSettings { /// withdraw or grant the right on every push. /// /// `builder_flows` lets every operator of this workspace compose flows out of already-deployed - /// runnables, and makes each of them consume a full author seat instead of half of one. + /// runnables, `builder_apps` full-code apps. Either makes each operator consume a full author + /// seat instead of half of one. #[serde(default, skip_serializing_if = "Option::is_none")] builder_flows: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + builder_apps: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] manage_schedules: Option, #[serde(default, skip_serializing_if = "Option::is_none")] manage_triggers: Option, @@ -11626,9 +11641,10 @@ async fn update_operator_settings( require_admin(authed.is_admin, &authed.username)?; // Every operator of the workspace turns into a full seat, which an offline license may not - // cover. It is a no-op delta when the right is already on. + // cover. Either right costs the same seat, so this prices "any", and it is a no-op delta when + // one of them is already on. #[cfg(feature = "enterprise")] - if settings.builder_flows == Some(true) { + if settings.builder_flows == Some(true) || settings.builder_apps == Some(true) { if let Some(msg) = windmill_common::ee_oss::check_seat_cap_for_operator_builder(&db, &w_id).await? { diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 50833bfe06..2f0d4deb13 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -1299,6 +1299,13 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + sqlx::query!( + "DELETE FROM runnable_job_stats WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + // If this workspace is itself a dev workspace, deleting it dissolves the pairing, so also drop // the parent prod's reserved dev_workspace_lock (mirrors detach_dev_workspace) — otherwise prod // stays locked against direct deploy/forking with no dev workspace left to make changes in. diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index ad7f12453c..b3aea64f72 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -20,7 +20,7 @@ benchmark = [] embedding = ["windmill-api-embeddings/embedding"] parquet = ["dep:datafusion", "windmill-common/parquet", "windmill-object-store/parquet", "windmill-worker?/parquet", "windmill-indexer?/parquet", "windmill-api-users/parquet", "windmill-api-settings/parquet", "windmill-api-workspaces/parquet", "windmill-api-npm-proxy/parquet", "dep:aws-sigv4", "dep:aws-sdk-config", "dep:quick-xml"] prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker?/prometheus", "windmill-api-scripts/prometheus"] -openidconnect = ["dep:openidconnect", "windmill-common/openidconnect", "windmill-store/openidconnect"] +openidconnect = ["dep:openidconnect", "dep:rsa", "dep:p256", "dep:tokio-rustls", "windmill-common/openidconnect", "windmill-store/openidconnect"] # The service log search handler reads the columnar store, so tantivy alone is # not enough for this crate to build on its own. tantivy = ["dep:windmill-indexer", "parquet"] @@ -141,6 +141,8 @@ samael = { workspace = true, optional = true } libxml = { workspace = true, optional = true } async-recursion.workspace = true rsa = { workspace = true, optional = true} +p256 = { workspace = true, optional = true } +tokio-rustls = { workspace = true, optional = true } uuid.workspace = true datafusion = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} @@ -189,4 +191,7 @@ flate2.workspace = true strum = { workspace = true, optional = true } dashmap.workspace = true +[dev-dependencies] +rcgen.workspace = true + [build-dependencies] diff --git a/backend/windmill-api/openapi-deref.json b/backend/windmill-api/openapi-deref.json index 8dae1bafcd..4016222d30 100644 --- a/backend/windmill-api/openapi-deref.json +++ b/backend/windmill-api/openapi-deref.json @@ -34993,7 +34993,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -35006,7 +35006,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -47930,7 +47930,7 @@ }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -47943,7 +47943,7 @@ "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 439e4d11a0..d541144231 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -11725,9 +11725,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: >- - Retry with exponential backoff (delay - doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -11738,7 +11736,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -23272,7 +23270,7 @@ paths: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier × seconds^n' properties: attempts: type: integer @@ -23283,7 +23281,7 @@ paths: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 47e617a524..8b5edc12cb 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.826.0 + version: 1.830.0 title: Windmill API contact: @@ -1765,6 +1765,12 @@ paths: type: string cluster: $ref: "#/components/schemas/DatatableRoleCluster" + take_over: + type: boolean + description: > + manage a Postgres role of that name that already exists on the cluster, + replacing its password, instead of refusing it. A role with cluster-wide + attributes or memberships is refused either way responses: "200": description: the created role @@ -1772,6 +1778,8 @@ paths: application/json: schema: $ref: "#/components/schemas/InstanceDatatableRole" + "409": + description: a Postgres role of that name already exists on the cluster and take_over was not set /settings/datatable_roles/{id}: post: @@ -1911,7 +1919,7 @@ paths: /settings/instance_ui: get: - summary: get the instance-wide announcement banner and accent color + summary: get the instance-wide announcement banner, accent color and API base url description: | Readable by any authenticated user; every page load reads it. Each value is the raw stored setting, or null when unset. @@ -1920,7 +1928,7 @@ paths: - setting responses: "200": - description: the instance_banner and accent_color settings + description: the instance_banner, accent_color and api_base_url settings content: application/json: schema: @@ -1928,6 +1936,7 @@ paths: properties: instance_banner: {} accent_color: {} + api_base_url: {} /settings_u/ruff_config: get: @@ -5559,6 +5568,9 @@ paths: type: boolean default_role: type: string + description: >- + The id of one of `roles`, `admin` (the default), or `$none` for no default + role: a connection that names no role then holds its viewer's roles only. roles: type: array items: @@ -7427,6 +7439,72 @@ paths: items: $ref: "#/components/schemas/TruncatedToken" + /oidc/signing_status: + get: + summary: what signs OIDC tokens and the keys the JWKS serves (ee only, superadmin) + operationId: getOidcSigningStatus + tags: + - oidc + responses: + "200": + description: OIDC signing status + content: + application/json: + schema: + type: object + required: + - mode + - keys + properties: + mode: + type: string + enum: [database, file, external] + keys: + type: array + items: + type: object + required: + - alg + - state + - source + properties: + kid: + type: string + alg: + type: string + state: + type: string + enum: [pending, active, retired, published] + source: + type: string + enum: [signer, database, file, publish_dir] + signer: + type: object + description: set when an external signer is configured + required: + - breaker + - max_rps + - current_rps + properties: + last_refresh_success: + type: string + format: date-time + last_refresh_error: + type: string + breaker: + type: string + enum: [closed, open, half_open] + max_rps: + type: number + current_rps: + type: number + keys_from_store: + type: boolean + description: the keys served are those stored by the last accepted listing, the signer has not answered this server yet + tls_files_error: + type: string + description: why the mTLS files on disk are not the ones in use + /w/{workspace}/oidc/token/{audience}: post: summary: get OIDC token (ee only) @@ -10916,6 +10994,37 @@ paths: description: "`saved` only, upsert or delete: where the write landed. Differs from the URL path when the item had moved away from it; the editor follows it there. Absent when a delete found nothing to remove and the caller cannot read the path it moved to." required: [status, current_timestamp] + /w/{workspace}/drafts/delete_for_user/{kind}/{path}: + post: + summary: delete another user's draft, or every draft at a path (admin only) + description: Delete the draft a given user holds at a path, or, without `username`, every draft at that path (the legacy ownerless one and the caller's own included). Workspace admins / superadmins only, and only for script, flow, app and raw_app drafts. Unlike a discard of one's own draft through `drafts/update`, this returns 404 when nothing was deleted. + operationId: deleteDraftForUser + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + schema: + $ref: "#/components/schemas/UserDraftItemKind" + - $ref: "#/components/parameters/ScriptPath" + - name: username + in: query + required: false + description: Workspace username of the draft owner. Omit to delete every user's draft at the path. + schema: + type: string + responses: + "200": + description: draft(s) deleted + content: + text/plain: + schema: + type: string + "404": + description: no matching draft at that path + /w/{workspace}/drafts/move/{kind}/{path}: post: summary: move the current user's draft-only item to another path @@ -12004,6 +12113,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) in: query @@ -16008,6 +16118,7 @@ paths: - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/RunRetry" - $ref: "#/components/parameters/IncludeHeader" - name: invisible_to_owner description: make the run invisible to the the script owner (default false) @@ -17312,6 +17423,27 @@ paths: schema: type: string + /w/{workspace}/jobs/worker_is_draining/{id}: + get: + summary: whether the worker running this job is draining + description: | + True once the worker running the job has received its shutdown signal. The worker + still waits for the job to finish, so a long-running job can poll this to exit early. + False when the job is not currently running on a worker. + operationId: workerIsDraining + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: whether the worker running this job is draining + content: + application/json: + schema: + type: boolean + /w/{workspace}/jobs_u/get_logs/{id}: get: summary: get job logs @@ -23492,6 +23624,54 @@ paths: - id - values + /workers/runnable_stats: + get: + summary: get resource usage per runnable and worker group over a time window (require devops role) + operationId: getRunnableStats + tags: + - worker + parameters: + - name: window_secs + in: query + required: false + description: how far back to read, in seconds (defaults to one day, capped at the 30-day retention) + schema: + type: integer + - name: worker_group + in: query + required: false + description: only the jobs run by this worker group + schema: + type: string + - name: workspace + in: query + required: false + description: only the jobs of this workspace + schema: + type: string + - name: order_by + in: query + required: false + description: "what the rows are ranked by, descending (default: total_duration)" + schema: + type: string + enum: [total_duration, max_memory, cpu] + - name: limit + in: query + required: false + description: number of rows to return (default 100, at most 1000) + schema: + type: integer + responses: + "200": + description: resource usage per runnable and worker group, heaviest first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/RunnableStats" + /workers/queue_metrics_series: get: summary: get the queue metrics of a time window, as a bounded line per tag @@ -25199,9 +25379,12 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Parquet Preview + description: > + Parquet Preview: `columns`, `column_types` (Arrow type names) and `rows`. content: application/json: {} @@ -25226,6 +25409,13 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - name: csv_separator + in: query + schema: + type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": description: Table count @@ -25274,13 +25464,20 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" - name: csv_separator in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Csv Preview + description: > + Csv Preview: `columns`, `column_types` (Arrow type names), `rows`, and the + `csv_separator` the file was read with, guessed from its first bytes when + none is given. + A `.jsonl` or `.ndjson` file is read as newline-delimited JSON. content: application/json: {} @@ -27640,6 +27837,28 @@ components: required: true schema: type: string + TablePreviewS3ResourcePath: + name: s3_resource_path + description: Read the file from this object storage resource instead of the workspace storage. + in: query + schema: + type: string + TablePreviewCsvHasHeader: + name: csv_has_header + description: Whether the first CSV line names the columns. Defaults to true. + in: query + schema: + type: boolean + TablePreviewFilters: + name: filters + description: > + JSON object, column name to filter. A number column takes an optional comparison + operator and a number (`>=10`), a boolean column a boolean, and any other column a + case-insensitive substring, or `=value` for an exact match. A `search_term` without + a `search_col` matches any column, case-insensitively. + in: query + schema: + type: string HubPublishFolder: name: folder in: query @@ -27822,6 +28041,12 @@ components: in: query schema: type: string + RunRetry: + name: retry + description: JSON-encoded retry policy (same shape as a schedule's retry) re-running the job when it fails. The returned id is the first attempt's; each retry is a job of its own, recorded as a child of the first attempt. Not supported for hub scripts, with invisible_to_owner, or for a script whose preprocessor would run. + in: query + schema: + type: string NewJobId: name: job_id description: @@ -31694,6 +31919,7 @@ components: - "resource_types.create" - "resource_types.update" - "resource_types.delete" + - "resource_types.sync" - "schedule.create" - "schedule.setenabled" - "schedule.edit" @@ -34525,6 +34751,11 @@ components: type: boolean cluster: $ref: "#/components/schemas/DatatableRoleCluster" + in_use: + type: boolean + description: >- + Whether any data table, in any workspace, lets someone connect as this role. Only set + in a data table's available_roles. DatatableRoleTenants: type: object @@ -34554,6 +34785,7 @@ components: type: boolean default_role: type: string + description: A role id, `admin`, or `$none` when the data table has no default role. roles: type: array items: @@ -35832,6 +36064,45 @@ components: `u/`, `g/`, or an email that exists in this workspace. + RunnableStats: + type: object + description: > + Resource usage of the jobs of one runnable on one worker group. Memory and CPU are + only known for jobs run as a subprocess; memory is the peak of the job's main process. + properties: + workspace_id: + type: string + runnable_path: + type: string + description: path of the script or flow step; `` gathers previews and other runs without a stable path + worker_group: + type: string + job_count: + type: integer + total_duration_ms: + type: integer + max_memory_peak: + type: integer + description: highest peak memory of a job, in kB + sum_memory_peak: + type: integer + description: sum of the peak memory of the jobs that have one, in kB + memory_sample_count: + type: integer + description: number of jobs that have a peak memory reading + total_cpu_ms: + type: integer + required: + - workspace_id + - runnable_path + - worker_group + - job_count + - total_duration_ms + - max_memory_peak + - sum_memory_peak + - memory_sample_count + - total_cpu_ms + WorkerPing: type: object properties: @@ -35880,6 +36151,9 @@ components: type: string native_mode: type: boolean + draining: + type: boolean + description: the worker received its shutdown signal and exits once its current job is done required: - worker - worker_instance @@ -37062,6 +37336,7 @@ components: default_role, can_create_schema, creatable_schemas, + migrations_enabled, ] properties: datatable_name: @@ -37095,6 +37370,9 @@ components: description: the schemas the role the listing connected as may create in items: type: string + migrations_enabled: + type: boolean + description: whether its schema is kept as versioned migrations, which a change made with plain SQL would bypass DataTableTableSchema: type: object @@ -37842,6 +38120,9 @@ components: builder_flows: type: boolean description: Whether operators can compose flows out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged. + builder_apps: + type: boolean + description: Whether operators can compose full-code apps out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged. manage_schedules: type: boolean description: Whether operators can create, edit and delete schedules. Granted unless withdrawn; omitting the field leaves the stored value unchanged. diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index c401568868..cf7447ceba 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -40,6 +40,7 @@ use windmill_ai::proxy::{ fim::maybe_transform_fim_request, proxy_execution_mode, ProxyBuildArgs, ProxyExecutionMode, ProxyRequest, }; +use windmill_ai::retry::send_with_retries; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; @@ -1188,15 +1189,11 @@ async fn global_proxy( } }; - let response = request.send().await.map_err(to_anyhow)?; + let response = send_with_retries(request).await; audit_global_ai_request(&db, &authed).await?; - if response.error_for_status_ref().is_err() { - let err_msg = response.text().await.unwrap_or("".to_string()); - return Err(Error::AIError(err_msg)); - } - + let response = response?; let status_code = response.status(); let headers = response.headers().clone(); let stream = response.bytes_stream(); @@ -1576,7 +1573,7 @@ async fn proxy( } }; - let response = request.send().await.map_err(to_anyhow)?; + let response = send_with_retries(request).await; let mut tx = db.begin().await?; @@ -1592,11 +1589,7 @@ async fn proxy( .await?; tx.commit().await?; - if response.error_for_status_ref().is_err() { - let err_msg = response.text().await.unwrap_or("".to_string()); - return Err(Error::AIError(err_msg)); - } - + let response = response?; let status_code = response.status(); let headers = response.headers().clone(); let is_sse = is_sse_response(&headers); diff --git a/backend/windmill-api/src/ai_sessions.rs b/backend/windmill-api/src/ai_sessions.rs index 65a8137c4e..9e980a54f4 100644 --- a/backend/windmill-api/src/ai_sessions.rs +++ b/backend/windmill-api/src/ai_sessions.rs @@ -24,7 +24,7 @@ use axum::{ Extension, Json, Router, }; use futures::{StreamExt, TryStreamExt}; -use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use magic_crypt::MagicCryptTrait; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use std::sync::Arc; @@ -35,7 +35,7 @@ use windmill_api_workspaces::ai_session_backups::{ use windmill_api_workspaces::workspaces::sessions_retention_days; use windmill_common::error::{Error, JsonResult, Result}; use windmill_common::utils::calculate_hash; -use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key}; +use windmill_common::variables::{crypt_from_key_with_suffix, get_workspace_key, WorkspaceCrypt}; use windmill_object_store::object_store_reexports::{ ObjectStore, ObjectStoreError, Path as ObjectPath, PutPayload, }; @@ -135,7 +135,7 @@ enum Read { /// The user's prefix in the workspace storage, plus what reads and writes it. struct Backend { store: Arc, - mc: MagicCrypt256, + mc: WorkspaceCrypt, prefix: String, /// Name the storage and the generation the objects are under, for the browser's sync /// state: a row recorded against another storage or generation is stale, a removal is diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index f276e6d143..2c2179d830 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -56,7 +56,10 @@ use std::str; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ - apps::{AppScriptId, ListAppQuery, APP_WORKSPACED_ROUTE}, + apps::{ + app_value_has_inline_script, app_value_runnable_paths, AppScriptId, ListAppQuery, + APP_WORKSPACED_ROUTE, + }, auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN}, cache::{self, future::FutureCachedExt}, db::{DbWithOptAuthed, UserDB}, @@ -75,7 +78,8 @@ use windmill_common::{ variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, workspaces::{ - check_deploy_rules, check_user_against_rule, ProtectionRuleKind, RuleCheckResult, + check_deploy_rules, check_operator_can_build, check_user_against_rule, + operator_builder_rights, BuilderKind, ProtectionRuleKind, RuleCheckResult, }, HUB_BASE_URL, }; @@ -749,7 +753,7 @@ async fn list_apps( // Append the authed user's `app`/`raw_app` drafts at paths with no deployed app; // see scripts.rs. if lq.include_draft_only.unwrap_or(false) - && !authed.is_operator + && (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.apps) && offset == 0 && lq.path_start.is_none() && lq.path_exact.is_none() @@ -979,7 +983,14 @@ async fn get_raw_app_data( // the opaque, sandboxed wrapper (incl. under a cross-origin-isolated // / COEP `require-corp` embedder). .header("X-Content-Type-Options", "nosniff") - .header("Cross-Origin-Resource-Policy", "cross-origin"); + .header("Cross-Origin-Resource-Policy", "cross-origin") + // The secret names one app version, whose bundle is written once at deploy, + // so the URL's content never changes. `private`: the URL is a capability, so + // shared caches must not keep serving it after the app is deleted. + .header( + http::header::CACHE_CONTROL, + "private, max-age=31536000, immutable", + ); Ok(res.body(body).unwrap()) } else { return Err(Error::NotFound("File not found".to_string())); @@ -1733,11 +1744,14 @@ async fn mint_preview_sdk_token( Path(w_id): Path, Json(req): Json, ) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot preview raw apps".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "preview raw apps", + ) + .await?; check_scopes(&authed, || format!("apps:write:{}", req.path))?; let (token, _expiration) = mint_raw_app_sdk_token(&db, &w_id, &req.path, &authed, &req.scopes, job_id).await?; @@ -2095,18 +2109,23 @@ async fn get_id_from_secret( prefix: Option<&str>, ) -> Result { let mc = build_crypt(db, w_id).await?; - let decrypted = mc - .decrypt_bytes_to_bytes(&(hex::decode(secret)?)) - .map_err(|e| Error::internal_err(e.to_string()))?; - let mut bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?; - if let Some(prefix) = prefix { - if !bytes.starts_with(prefix) { - return Err(Error::BadRequest("Invalid secret".to_string())); - } - bytes = bytes.strip_prefix(prefix).unwrap_or(""); + id_from_secret(&mc, &secret, prefix) + .ok_or_else(|| Error::BadRequest("Invalid secret".to_string())) +} + +/// The secret comes from the URL of unauthenticated routes, so every way it can be invalid +/// must produce the same response: a caller must never learn why a secret was rejected. +fn id_from_secret( + mc: &magic_crypt::MagicCrypt256, + secret: &str, + prefix: Option<&str>, +) -> Option { + let decrypted = mc.decrypt_bytes_to_bytes(&hex::decode(secret).ok()?).ok()?; + let plaintext = String::from_utf8(decrypted).ok()?; + match prefix { + Some(prefix) => plaintext.strip_prefix(prefix)?.parse().ok(), + None => plaintext.parse().ok(), } - let id: i64 = bytes.parse().map_err(to_anyhow)?; - Ok(id) } async fn get_public_resource( @@ -2255,6 +2274,260 @@ async fn store_raw_app_file<'a>( Ok(()) } +/// A builder app write is held to one set of rules whether it deploys or saves a draft: the +/// editor previews a draft as whoever opens it, and an admin may deploy it as it stands. A deploy +/// fills in what a draft must state, since it stores the policy it checked. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum BuilderAppWrite { + Deploy, + Draft, +} + +/// The half of the builder-app check that needs no DB (see `docs/operator-builder-rights.md`). +/// Returns every runnable the app can invoke, as `(is_flow, path)`, from both the policy's +/// triggerables and the value's path runnables: they are not the same list. +fn check_operator_composed_app( + write: BuilderAppWrite, + raw_app: bool, + value: Option<&RawValue>, + policy: Option<&mut Policy>, +) -> Result> { + let verb = if write == BuilderAppWrite::Deploy { + "deploy" + } else { + "save" + }; + if !raw_app { + return Err(Error::PermissionDenied( + "Operators with builder rights can only author full-code apps".to_string(), + )); + } + let mut referenced: Vec<(bool, String)> = Vec::new(); + if let Some(value) = value { + // The text stored, NULs stripped: a stripped NUL can rename a key into one this reads. + let value: serde_json::Value = + serde_json::from_str(&strip_json_nul(value.get())).map_err(to_anyhow)?; + if app_value_has_inline_script(&value) { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot {verb} an app carrying inline scripts" + ))); + } + referenced.extend(app_value_runnable_paths(&value)?); + } + let Some(policy) = policy else { + return Err(match write { + BuilderAppWrite::Deploy => Error::BadRequest( + "Operators with builder rights must deploy an app with its policy".to_string(), + ), + BuilderAppWrite::Draft => Error::PermissionDenied( + "Operators with builder rights can only save sandboxed apps".to_string(), + ), + }); + }; + // Every deploy of a draft rebuilds `triggerables_v2` from the runnables, so a draft's copy, + // stale since the last deploy, must not block autosave. A deploy path that skips the rebuild + // must stop exempting it here. + if write == BuilderAppWrite::Draft { + policy.triggerables_v2 = None; + } + referenced.extend(checked_policy_runnable_paths(policy)?); + match (policy.sandbox, write) { + (Some(true), _) => {} + (None, BuilderAppWrite::Deploy) => policy.sandbox = Some(true), + _ => { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights can only {verb} sandboxed apps" + ))) + } + } + + // Viewer mode would make the triggerables collected above non-exhaustive (see the docs), and + // the deploy panel shows it as members-only, so an admin deploying a draft would not see it. + // Pin an omitted mode on deploy: `update_app_internal` resolves it to the deployed app's, so a + // redeploy over an admin's viewer-mode app would otherwise inherit `Viewer`. + match policy.stated_execution_mode() { + Some(ExecutionMode::Viewer) => { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot {verb} an app that runs as its viewer" + ))) + } + None if write == BuilderAppWrite::Deploy => { + policy.set_execution_mode(ExecutionMode::Publisher) + } + _ => {} + } + + referenced.sort(); + referenced.dedup(); + refuse_hub_runnables(&referenced)?; + Ok(referenced) +} + +/// The runnables a builder app's policy authorizes, as `(is_flow, path)`; refuses a policy that +/// pins inline code. +fn checked_policy_runnable_paths(policy: &Policy) -> Result> { + // A `rawscript/` triggerable is the deployed app's authorization to run caller-supplied + // `raw_code` hashing to it: pinning one hands a builder arbitrary code execution through an + // app whose value passed the inline-script check. A composition-only app has none. + fn pins_raw_script(map: &Option>) -> bool { + map.iter() + .flat_map(|m| m.keys()) + .any(|k| k.starts_with("rawscript/") || k.contains(":rawscript/")) + } + if pins_raw_script(&policy.triggerables) || pins_raw_script(&policy.triggerables_v2) { + return Err(Error::PermissionDenied( + "Operators with builder rights cannot save or deploy an app whose policy pins inline code" + .to_string(), + )); + } + // `execute_component` looks up `{component}:{path}` with an unrestricted component, so every + // colon is a possible split (`a:b:script/x` resolves for `component = "a:b"`): check every + // suffix that parses as a runnable. That copies a key once per colon, so bound the key first: + // a 255-char path plus a component name is far below the cap. + const MAX_KEY_LEN: usize = 512; + let keys = || { + policy + .triggerables + .iter() + .flat_map(|t| t.keys()) + .chain(policy.triggerables_v2.iter().flat_map(|t| t.keys())) + }; + if keys().any(|k| k.len() > MAX_KEY_LEN) { + return Err(Error::BadRequest(format!( + "App policy triggerable keys must be at most {MAX_KEY_LEN} bytes" + ))); + } + Ok(keys() + .flat_map(|key| { + std::iter::once(key.as_str()) + .chain(key.match_indices(':').map(|(i, _)| &key[i + 1..])) + .filter_map(|candidate| { + candidate + .strip_prefix("script/") + .map(|p| (false, p.to_string())) + .or_else(|| { + candidate + .strip_prefix("flow/") + .map(|p| (true, p.to_string())) + }) + }) + .collect::>() + }) + .collect()) +} + +fn refuse_hub_runnables(referenced: &[(bool, String)]) -> Result<()> { + for (_, path) in referenced { + if path.starts_with("hub/") { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot reference the hub runnable {path}. Deploy it to the workspace first." + ))); + } + } + Ok(()) +} + +/// Runs on every app write by an operator with builder rights, from inside the create/update +/// internals so both raw-app endpoints are covered by one check. +/// +/// `execute_component` resolves the runnable it picks with the root DB handle, so a path the +/// builder cannot read would still run, and in a `Viewer` app it would run as whoever opened it. +/// RLS on this transaction is the check; refusing `Viewer` above is what makes it exhaustive. +pub(crate) async fn validate_operator_composed_app( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + write: BuilderAppWrite, + raw_app: bool, + value: Option<&RawValue>, + policy: Option<&mut Policy>, +) -> Result<()> { + // A deploy resets the run identity to its builder (unless they are in `wm_deployers`, see + // docs/operator-builder-rights.md); a draft keeps whatever it names, which an admin's + // one-click draft deploy preserves without showing it. + if let (BuilderAppWrite::Draft, Some(policy)) = (write, policy.as_deref()) { + let names_another = policy + .on_behalf_of + .as_deref() + .is_some_and(|p| p != username_to_permissioned_as(&authed.username)) + || policy + .on_behalf_of_email + .as_deref() + .is_some_and(|e| e != authed.email); + if names_another { + return Err(Error::PermissionDenied( + "Operators with builder rights cannot save an app that runs as someone else" + .to_string(), + )); + } + } + let referenced = check_operator_composed_app(write, raw_app, value, policy)?; + require_runnables_readable(authed, db, user_db, w_id, referenced).await +} + +async fn require_runnables_readable( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + referenced: Vec<(bool, String)>, +) -> Result<()> { + if referenced.is_empty() { + return Ok(()); + } + // Each script version carries its own grants, and a path reused after its versions were + // archived starts with none. Check the version a run would pick, found without RLS, not + // whichever version the caller happens to see. Resolved before the transaction opens so no + // request holds two pool connections at once. + let mut hashes = Vec::with_capacity(referenced.len()); + for (is_flow, path) in &referenced { + hashes.push(if *is_flow { + None + } else { + windmill_common::get_latest_script_hash(db, path, w_id).await? + }); + } + let mut tx = user_db.clone().begin(authed).await?; + for ((is_flow, path), hash) in referenced.into_iter().zip(hashes) { + let readable = if is_flow { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2)", + w_id, + path, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false) + } else { + match hash { + Some(hash) => sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)", + w_id, + hash, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false), + None => false, + } + }; + // One message for every cause: telling "no runnable version" apart from "not readable" + // would reveal that a path the caller cannot read exists. + if !readable { + return Err(Error::PermissionDenied(if is_flow { + format!("Flow {path} does not exist or is not readable by you") + } else { + format!( + "Script {path} does not exist, is not readable by you, or has not finished deploying" + ) + })); + } + } + tx.commit().await?; + Ok(()) +} + macro_rules! process_app_multipart { ($authed:expr, $user_db:expr, $db:expr, $w_id:expr, $path:expr, $multipart:expr, $internal_fn:expr) => { async { @@ -2331,11 +2604,14 @@ async fn create_app_raw<'a>( Path(w_id): Path, multipart: Multipart, ) -> Result<(StatusCode, String)> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot create apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "create apps", + ) + .await?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, @@ -2512,6 +2788,19 @@ async fn create_app_internal<'a>( if raw_app { validate_raw_app_path_keys(&app.value.0)?; } + if authed.is_operator { + validate_operator_composed_app( + &authed, + &db, + &user_db, + w_id, + BuilderAppWrite::Deploy, + raw_app, + Some(&app.value.0), + Some(&mut app.policy), + ) + .await?; + } if *CLOUD_HOSTED { let nb_apps = sqlx::query_scalar!("SELECT COUNT(*) FROM app WHERE workspace_id = $1", &w_id) @@ -2818,13 +3107,23 @@ async fn delete_app( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot delete apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "delete apps", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("apps:write:{}", path))?; + // One endpoint deletes both kinds, and builder rights only cover full-code apps. + if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only delete full-code apps".to_string(), + )); + } if path == "g/all/setup_app" && w_id == "admins" { return Err(Error::BadRequest( @@ -3012,13 +3311,20 @@ async fn update_app( Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, ) -> JsonResult { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update apps for security reasons".to_string(), - )); - } - // create_app_internal(authed, user_db, db, &w_id, &mut app).await?; let path = path.to_path(); + // The deploy panel saves a deployed full-code app's access and frontend scopes here. A + // builder may change those settings; the app itself only through `update_app_raw`. + if authed.is_operator { + check_operator_can_build(&db, &w_id, true, BuilderKind::Apps, "update apps").await?; + if ns.value.is_some() + || deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only change the settings of full-code apps" + .to_string(), + )); + } + } check_scopes(&authed, || format!("apps:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( @@ -3035,8 +3341,10 @@ async fn update_app( let opath = path.to_string(); let db2 = db.clone(); + // Settings-only for an operator, of an app checked above to be full-code. + let raw_app = authed.is_operator; let (new_tx, npath, v_id) = - update_app_internal(authed, db, user_db, &w_id, path, false, ns, None).await?; + update_app_internal(authed, db, user_db, &w_id, path, raw_app, ns, None).await?; new_tx.commit().await?; tally_app_rename(&db2, &w_id, &opath, &npath, v_id).await; @@ -3065,6 +3373,10 @@ async fn update_app_raw_source( Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, ) -> JsonResult { + // Stays closed to operators even with builder rights, unlike `create_app_raw`/`update_app_raw` + // next to it: this path compiles caller-supplied sources with a bundler job on a worker, which + // is arbitrary code execution. Builders lose nothing: the browser and the CLI both bundle + // locally and deploy through the multipart endpoints. if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3225,6 +3537,7 @@ async fn create_app_raw_source( Path(w_id): Path, Json(mut app): Json, ) -> Result<(StatusCode, String)> { + // Closed to builders too, for the reason on `update_app_raw_source`. if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot create apps for security reasons".to_string(), @@ -3390,11 +3703,14 @@ async fn update_app_raw<'a>( Path((w_id, path)): Path<(String, StripPath)>, multipart: Multipart, ) -> JsonResult { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "update apps", + ) + .await?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, @@ -3410,6 +3726,14 @@ async fn update_app_raw<'a>( let path = path.to_path(); check_scopes(&authed, || format!("apps:write:{}", path))?; + // `update_app_internal` only compares kinds when a new value is deployed, so a + // policy-only update would otherwise let a builder rewrite a low-code app's policy. + if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only update full-code apps".to_string(), + )); + } let opath = path.to_string(); let db2 = db.clone(); let (npath, v_id) = process_app_multipart!( @@ -3477,6 +3801,23 @@ async fn update_app_internal<'a>( } } + if authed.is_operator { + // Restoring a full-code version always sends allow_kind_change. A builder converts + // nothing, so drop it and let the locked kind check below refuse an actual conversion. + ns.allow_kind_change = None; + validate_operator_composed_app( + &authed, + &db, + &user_db, + w_id, + BuilderAppWrite::Deploy, + raw_app, + ns.value.as_ref().map(|v| v.0.as_ref()), + ns.policy.as_mut(), + ) + .await?; + } + // Resolved on the (non-RLS) pool before the RLS transaction opens, for the reason // `create_app` states, with the same known, accepted rename race (see // `resolve_on_behalf_of`). Submitting a policy is how a deployer claims the app's execution @@ -4052,10 +4393,55 @@ lazy_static! { /// change or deletion, which don't bump the version — see /// `invalidate_app_policy_cache` and `process_notify_event`); /// - the entry is within its TTL (backstop for a missed event or a restart). - static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = + static ref APP_POLICY_CACHE: cache::Cache<(String, String), (Option, Arc, std::time::Instant)> = cache::Cache::new(1000); } +/// What `execute_component` runs a deployed app against. +struct DeployedPolicy { + policy: Policy, + /// Runnable id -> argument -> context property (`username`, `email`, `groups`, + /// `workspace`, `author`), for every `ctx` field the app's runnables declare. Read + /// from the deployed app rather than taken from the request, which only says what + /// the browser chose to send: a runnable filters rows on these. A deploy that + /// changes them under an unchanged policy evicts nothing, so a caller still naming + /// the replaced version keeps the previous declarations until the entry's TTL. + ctx_inputs: HashMap>, +} + +async fn deployed_ctx_inputs( + db: &DB, + w_id: &str, + path: &str, +) -> Result>> { + let runnables = sqlx::query_scalar::<_, Option>>( + "SELECT app_version.value -> 'runnables' FROM app + JOIN app_version ON app_version.id = app.versions[array_upper(app.versions, 1)] + WHERE app.path = $1 AND app.workspace_id = $2", + ) + .bind(path) + .bind(w_id) + .fetch_optional(db) + .await? + .flatten(); + let Some(serde_json::Value::Object(runnables)) = runnables.map(|r| r.0) else { + return Ok(HashMap::new()); + }; + Ok(runnables + .into_iter() + .filter_map(|(id, runnable)| { + let ctx_inputs: HashMap = runnable + .get("fields")? + .as_object()? + .iter() + .filter(|(_, f)| f.get("type").and_then(|t| t.as_str()) == Some("ctx")) + .filter_map(|(arg, f)| Some((arg.clone(), f.get("ctx")?.as_str()?.to_string()))) + .collect(); + (!ctx_inputs.is_empty()).then_some((id, ctx_inputs)) + }) + .collect()) +} + /// Backstop time-to-live for an [`APP_POLICY_CACHE`] entry — the maximum a policy /// change can go unreflected if its invalidation event is never consumed. const APP_POLICY_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(300); @@ -4108,7 +4494,7 @@ async fn execute_component( _ => {} }; - let arc_policy: Arc; + let arc_policy: Arc; let policy_triggerables_default = Default::default(); // Preview mode means the request was issued from the editor; the editing // user is already trusted by the policy check, so client-supplied `tag` @@ -4133,10 +4519,26 @@ async fn execute_component( let authed = opt_authed.as_ref().ok_or_else(|| { Error::NotAuthorized("App component preview requires authentication".to_string()) })?; + // A builder testing the app it is composing only ever previews a deployed runnable + // (`path`), confined below to what it may read. Inline `raw_code`, which an `id` also + // requires, is authoring code, so it stays closed to every operator. So is a hub + // path, which `require_path_read_access_for_preview` admits for everyone and + // `get_payload_tag_from_prefixed_path` then downloads and enqueues: it is exactly the + // unreviewed code the composition check refuses in a flow. + let previews_hub = payload.path.as_deref().is_some_and(|p| { + p.strip_prefix("script/") + .or_else(|| p.strip_prefix("flow/")) + .unwrap_or(p) + .starts_with("hub/") + }); if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + let msg = "Operators cannot run preview jobs for security reasons".to_string(); + if !operator_builder_rights(&db, &w_id).await?.apps { + return Err(Error::NotAuthorized(msg)); + } + if payload.raw_code.is_some() || previews_hub { + return Err(Error::PermissionDenied(msg)); + } } check_scopes(authed, || format!("jobs:run"))?; if let Some(p) = payload.path.as_deref() { @@ -4175,7 +4577,7 @@ async fn execute_component( // - The policy and triggerables are fetched from the database. // - In case of inline script, if an entry exists in the `app_script` table, push // an `AppScript` job payload, as in (.1) otherwise. - let (policy, policy_triggerables) = match payload { + let (policy, policy_triggerables, ctx_inputs) = match payload { // 1. "preview" mode. ExecuteApp { force_viewer_static_fields: Some(static_inputs), @@ -4194,6 +4596,7 @@ async fn execute_component( sensitive_inputs: force_viewer_sensitive_inputs.unwrap_or_default(), tag: None, }, + None, ), // 2. "run" mode. _ => { @@ -4223,9 +4626,12 @@ async fn execute_component( .map(|(_, p, _)| p); let policy = if let Some(p) = fresh { arc_policy = p; - &*arc_policy + &arc_policy.policy } else { - arc_policy = Arc::new(policy_fut.await?); + arc_policy = Arc::new(DeployedPolicy { + policy: policy_fut.await?, + ctx_inputs: deployed_ctx_inputs(&db, &w_id, path).await?, + }); APP_POLICY_CACHE.insert( cache_key, ( @@ -4234,7 +4640,7 @@ async fn execute_component( std::time::Instant::now(), ), ); - &*arc_policy + &arc_policy.policy }; // Caller-supplied inline code (`raw_code`), with or without an @@ -4296,7 +4702,11 @@ async fn execute_component( }) .ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?; - (policy, policy_triggerables) + ( + policy, + policy_triggerables, + arc_policy.ctx_inputs.get(&payload.component), + ) } }; @@ -4374,6 +4784,7 @@ async fn execute_component( let (mut args, job_id) = build_args( policy, policy_triggerables, + ctx_inputs, payload.args, opt_authed.as_ref(), &user_db, @@ -5491,6 +5902,10 @@ impl AppLoadCountQuery { search_col: self.search_col, search_term: self.search_term, storage: self.storage, + s3_resource_path: None, + csv_separator: None, + csv_has_header: None, + filters: None, }, ) } @@ -5524,6 +5939,9 @@ impl AppLoadPreviewQuery { search_term: self.search_term, storage: self.storage, csv_separator: self.csv_separator, + s3_resource_path: None, + csv_has_header: None, + filters: None, }, ) } @@ -5611,7 +6029,9 @@ async fn app_load_table_count( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = - crate::job_helpers_oss::load_table_count_internal(job_authed, &db, &w_id, file_key, inner) + crate::job_helpers_oss::load_table_count_internal( + job_authed, &db, None, &w_id, file_key, inner, + ) .await?; Ok(Json(resp).into_response()) } @@ -5630,7 +6050,7 @@ async fn app_load_parquet_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, true, + job_authed, &db, None, &w_id, file_key, inner, true, ) .await?; Ok(Json(resp).into_response()) @@ -5650,7 +6070,7 @@ async fn app_load_csv_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, false, + job_authed, &db, None, &w_id, file_key, inner, false, ) .await?; Ok(Json(resp).into_response()) @@ -5795,6 +6215,54 @@ fn policy_granted_viewer(authed: &ApiAuthed) -> ApiAuthed { ApiAuthed { scopes: None, ..authed.clone() } } +/// The value of an app context property for this run. Everything about the viewer comes +/// from `authed`, the session the request authenticated with; an anonymous run has none +/// and gets `null`. +async fn ctx_value( + prop: &str, + policy: &Policy, + authed: Option<&ApiAuthed>, + w_id: &str, + db: &DB, +) -> Result> { + let value = match prop { + "username" => authed.map(|a| serde_json::to_value(a.display_username())), + "email" => authed.map(|a| serde_json::to_value(&a.email)), + "workspace" => Some(serde_json::to_value(w_id)), + "groups" => authed.map(|a| serde_json::to_value(&a.groups)), + // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. + "author" => { + let author = match ( + policy.on_behalf_of_email.as_deref(), + policy.on_behalf_of.as_deref(), + ) { + (Some(email), _) => Some(email.to_string()), + (None, Some(permissioned_as)) => Some( + windmill_common::users::get_email_from_permissioned_as( + permissioned_as, + w_id, + db, + ) + .await?, + ), + (None, None) => None, + }; + Some(serde_json::to_value(&author)) + } + _ => { + return Err(Error::BadRequest(format!( + "context variable {} not allowed", + prop + ))) + } + }; + Ok(to_raw_value( + &value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { + Error::internal_err(format!("failed to serialize ctx variable {}: {}", prop, e)) + })?, + )) +} + async fn build_args( policy: &Policy, PolicyTriggerableInputs { @@ -5804,6 +6272,7 @@ async fn build_args( sensitive_inputs, .. }: &PolicyTriggerableInputs, + ctx_inputs: Option<&HashMap>, mut args: HashMap>, authed: Option<&ApiAuthed>, user_db: &UserDB, @@ -5813,6 +6282,16 @@ async fn build_args( let mut job_id: Option = None; let mut safe_args = HashMap::>::new(); + // Before anything reads `args`: a context input the runnable declares is never taken + // from the request, or a viewer could name someone else as themselves. + for (k, prop) in ctx_inputs.into_iter().flatten() { + args.remove(k); + safe_args.insert( + k.to_string(), + ctx_value(prop, policy, authed, w_id, db).await?, + ); + } + // tracing::error!("{:?}", allow_user_resources); for k in allow_user_resources.iter() { if let Some(arg_val) = args.get(k) { @@ -5904,47 +6383,9 @@ async fn build_args( if arg_str.starts_with("\"$ctx:") { let prop = arg_str.trim_start_matches("\"$ctx:").trim_end_matches("\""); - let value = match prop { - "username" => authed - .as_ref() - .map(|a| serde_json::to_value(a.display_username())), - "email" => authed.as_ref().map(|a| serde_json::to_value(&a.email)), - "workspace" => Some(serde_json::to_value(&w_id)), - "groups" => authed.as_ref().map(|a| serde_json::to_value(&a.groups)), - // Same rule as `get_on_behalf_of`: the stored address, derived only when absent. - "author" => { - let author = match ( - policy.on_behalf_of_email.as_deref(), - policy.on_behalf_of.as_deref(), - ) { - (Some(email), _) => Some(email.to_string()), - (None, Some(permissioned_as)) => Some( - windmill_common::users::get_email_from_permissioned_as( - permissioned_as, - w_id, - db, - ) - .await?, - ), - (None, None) => None, - }; - Some(serde_json::to_value(&author)) - } - _ => { - return Err(Error::BadRequest(format!( - "context variable {} not allowed", - prop - ))) - } - }; safe_args.insert( k.to_string(), - to_raw_value(&value.unwrap_or(Ok(serde_json::Value::Null)).map_err(|e| { - Error::internal_err(format!( - "failed to serialize ctx variable for {}: {}", - k, e - )) - })?), + ctx_value(prop, policy, authed, w_id, db).await?, ); } else if !arg_str.contains("\"$var:") && !arg_str.contains("\"$res:") { safe_args.insert(k.to_string(), v); @@ -6339,6 +6780,127 @@ mod embed_token_tests { } } +#[cfg(test)] +mod operator_app_tests { + use super::{check_operator_composed_app, BuilderAppWrite, Policy}; + use windmill_common::error::Result; + use windmill_common::worker::to_raw_value; + + fn builder_policy(triggerables_v2: serde_json::Value) -> Policy { + serde_json::from_value(serde_json::json!({ + "execution_mode": "publisher", + "triggerables_v2": triggerables_v2, + })) + .unwrap() + } + + fn path_runnable(path: &str, run_type: &str) -> serde_json::Value { + serde_json::json!({"files": {}, "runnables": { + "a": {"name": "a", "type": "runnableByPath", "path": path, "runType": run_type} + }}) + } + + fn composed_app(value: serde_json::Value, policy: &mut Policy) -> Result> { + let value = to_raw_value(&value); + check_operator_composed_app(BuilderAppWrite::Deploy, true, Some(&value), Some(policy)) + } + + #[test] + fn operator_app_check_forces_the_sandbox_and_refuses_code() { + let clean = serde_json::json!({"files": {}, "runnables": { + "a": {"name": "a", "type": "runnableByPath", "path": "f/x/s", "runType": "script"} + }}); + + // A composition-only app goes through, sandboxed whether or not it asked to be. + let mut policy = builder_policy( + serde_json::json!({"a:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + let referenced = composed_app(clean.clone(), &mut policy).unwrap(); + assert_eq!(policy.sandbox, Some(true)); + // The runnables the policy authorizes are handed back for the caller to authorize. + assert_eq!(referenced, vec![(false, "f/x/s".to_string())]); + + // A hub reference is unreviewed code, refused like it is in a flow. + let mut policy = builder_policy( + serde_json::json!({"a:script/hub/1/x": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + let mut policy = builder_policy(serde_json::json!({})); + policy.sandbox = Some(false); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + // An inline script is refused even with no `language`, which the locking traversal skips. + let mut policy = builder_policy(serde_json::json!({})); + assert!(composed_app( + serde_json::json!({"runnables": {"a": {"inlineScript": {"content": "x"}}}}), + &mut policy + ) + .is_err()); + + // A `rawscript/` triggerable is what authorizes caller-supplied `raw_code` on the + // deployed app, so pinning one would be arbitrary code execution behind a clean value. + for key in ["rawscript/abc", "a:rawscript/abc"] { + let mut policy = builder_policy( + serde_json::json!({ key: {"static_inputs": {}, "one_of_inputs": {}} }), + ); + assert!( + composed_app(clean.clone(), &mut policy).is_err(), + "{key} must be refused" + ); + } + + // What the deployed bundle actually asks to run comes from the value's `runnableByPath` + // entries, which are a separate surface from the policy: both are reported. + let mut policy = builder_policy(serde_json::json!({})); + assert_eq!( + composed_app(path_runnable("f/x/s", "script"), &mut policy).unwrap(), + vec![(false, "f/x/s".to_string())] + ); + let mut policy = builder_policy(serde_json::json!({})); + assert_eq!( + composed_app(path_runnable("f/x/f", "flow"), &mut policy).unwrap(), + vec![(true, "f/x/f".to_string())] + ); + let mut policy = builder_policy(serde_json::json!({})); + assert!(composed_app(path_runnable("hub/1/x", "hubscript"), &mut policy).is_err()); + + // `execute_component` looks up `:` with an unrestricted component, so a + // second colon must not hide the path from validation: `x:y:script/hub/…` resolves at run + // time for `component = "x:y"`. + let mut policy = builder_policy( + serde_json::json!({"x:y:script/hub/123/foo": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + let mut policy = builder_policy( + serde_json::json!({"x:y:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert_eq!( + composed_app(clean.clone(), &mut policy).unwrap(), + vec![(false, "f/x/s".to_string())] + ); + + // `Viewer` mode makes `execute_component` accept any script/flow path, triggerables or + // not, and run it as the viewer, so the checks above would stop binding. + let mut policy: Policy = serde_json::from_value(serde_json::json!({ + "execution_mode": "viewer", "triggerables_v2": {} + })) + .unwrap(); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + // Low-code apps stay closed. + let value = to_raw_value(&clean); + let mut policy = builder_policy(serde_json::json!({})); + assert!(check_operator_composed_app( + BuilderAppWrite::Deploy, + false, + Some(&value), + Some(&mut policy) + ) + .is_err()); + } +} + #[cfg(test)] mod policy_tests { /// `execution_mode` is optional in the OpenAPI schema the API clients and the @@ -6361,6 +6923,39 @@ mod policy_tests { } } +#[cfg(test)] +mod app_secret_tests { + use super::{id_from_secret, BUNDLE_SECRET_PREFIX}; + use magic_crypt::MagicCryptTrait; + + #[test] + fn every_invalid_secret_is_rejected_alike() { + let mc = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = magic_crypt::new_magic_crypt!("another-key", 256); + let valid = hex::encode(mc.encrypt_str_to_bytes("42")); + assert_eq!(id_from_secret(&mc, &valid, None), Some(42)); + let bundle = hex::encode(mc.encrypt_str_to_bytes(format!("{BUNDLE_SECRET_PREFIX}7"))); + assert_eq!( + id_from_secret(&mc, &bundle, Some(BUNDLE_SECRET_PREFIX)), + Some(7) + ); + + let mut tampered = mc.encrypt_str_to_bytes("42"); + *tampered.last_mut().unwrap() ^= 1; + for bad in [ + "not hex".to_string(), + hex::encode(&tampered), + hex::encode(other.encrypt_str_to_bytes("42")), + hex::encode(mc.encrypt_bytes_to_bytes(&[0xff, 0xfe])), + hex::encode(mc.encrypt_str_to_bytes("not a number")), + valid.clone(), + ] { + let prefix = (bad == valid).then_some(BUNDLE_SECRET_PREFIX); + assert_eq!(id_from_secret(&mc, &bad, prefix), None, "{bad}"); + } + } +} + #[cfg(test)] mod raw_app_file_key_tests { use super::validate_raw_app_path_keys; diff --git a/backend/windmill-api/src/apps_raw_bundler.ts b/backend/windmill-api/src/apps_raw_bundler.ts index 6663a0f83e..965a9289b5 100644 --- a/backend/windmill-api/src/apps_raw_bundler.ts +++ b/backend/windmill-api/src/apps_raw_bundler.ts @@ -6,7 +6,7 @@ * The build itself is `wmill app bundle`, the same one `wmill app push` runs, so * an app deployed through the API is compiled exactly as the CLI and the editor * compile it — entry point, virtual `wmill` module, `/ui/` shared-UI resolution - * and the Svelte/Vue plugins included. Reimplementing any of that here would be + * and the Svelte plugin included. Reimplementing any of that here would be * a third bundler to keep in step with the other two. */ declare const __wmillAppPolicy: { diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index a3cd34f3f8..e6b6db6919 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -15,15 +15,16 @@ use axum::{ }; use serde::{Deserialize, Serialize}; use windmill_api_flows::flows::validate_operator_flow; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{Error, Result}, flows::FlowValue, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, - utils::{check_proper_path, strip_json_nul}, + utils::{check_proper_path, require_admin, strip_json_nul}, variables::{build_crypt, encrypt}, - workspaces::operator_can_build_flows, + workspaces::{operator_builder_rights, BuilderKind}, }; pub fn workspaced_service() -> Router { @@ -32,6 +33,10 @@ pub fn workspaced_service() -> Router { .route("/get/{kind}/{*path}", get(get_draft_for_user)) .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) + .route( + "/delete_for_user/{kind}/{*path}", + post(delete_draft_for_user), + ) .route("/move/{kind}/{*path}", post(move_draft)) .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } @@ -107,7 +112,7 @@ async fn list_drafts( // Without builder rights an operator has no drafts of their own (they can't write any, see // `require_can_write_path`), so this list is always empty for them. They can still READ some // collaborators' drafts via `/drafts/get`. - if authed.is_operator && !operator_can_build_flows(&db, &w_id).await? { + if authed.is_operator && !operator_builder_rights(&db, &w_id).await?.any() { return Ok(Json(vec![])); } let all_users = query.all_users.unwrap_or(false); @@ -496,6 +501,31 @@ async fn update_draft( } } + // A raw-app draft carries the app value and its policy side by side. + if authed.is_operator && kind == UserDraftItemKind::RawApp { + if let Some(value) = &req.value { + let draft: serde_json::Value = serde_json::from_str(&strip_json_nul(value.0.get())) + .map_err(|e| Error::BadRequest(format!("Invalid app draft: {e}")))?; + let mut policy = draft + .get("policy") + .filter(|p| !p.is_null()) + .map(|p| serde_json::from_value::(p.clone())) + .transpose() + .map_err(|e| Error::BadRequest(format!("Invalid app draft policy: {e}")))?; + crate::apps::validate_operator_composed_app( + &authed, + &db, + &user_db, + &w_id, + crate::apps::BuilderAppWrite::Draft, + true, + Some(&value.0), + policy.as_mut(), + ) + .await?; + } + } + let applied = if let Some(value) = &req.value { // Secret variable values must never sit in `draft.value` in plaintext // (see `encrypt_secret_variable_value`). @@ -1000,6 +1030,85 @@ async fn migrate_legacy_draft( } } +#[derive(Deserialize, Debug)] +pub struct DeleteDraftForUserQuery { + /// Workspace username of the draft owner. Absent: every draft at the path, + /// the legacy ownerless one and the caller's own included. + pub username: Option, +} + +/// Delete ANOTHER user's draft at a path, or every draft there, for workspace +/// admins (and superadmins, which carry `is_admin` in a workspace). Covers the +/// kinds whose authors are visible to others (`shares_drafts_across_users`); the +/// owner is named by workspace username, as on `/drafts/get`. +async fn delete_draft_for_user( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, + Query(query): Query, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + if !kind.shares_drafts_across_users() { + return Err(Error::NotFound( + "drafts for this item kind are private to their owner".to_string(), + )); + } + let path = path.to_path(); + let username = query.username.as_deref(); + let owner_email = match username { + Some(username) => Some( + resolve_username_to_email(&w_id, username, &db) + .await? + .ok_or_else(|| { + Error::NotFound(format!("no user with username {username} in workspace")) + })?, + ), + None => None, + }; + + let mut tx = db.begin().await?; + let deleted = sqlx::query_scalar!( + r#"DELETE FROM draft + WHERE workspace_id = $1 AND path = $2 AND typ = $3 + AND ($4::text IS NULL OR email = $4) + RETURNING 1 as "one!""#, + &w_id, + path, + kind as UserDraftItemKind, + owner_email.as_deref(), + ) + .fetch_all(&mut *tx) + .await? + .len(); + if deleted == 0 { + return Err(Error::NotFound(match username { + Some(username) => format!("no draft for {username} at {path}"), + None => format!("no draft at {path}"), + })); + } + audit_log( + &mut *tx, + &authed, + "drafts.delete_for_user", + ActionKind::Delete, + &w_id, + Some(path), + Some( + [ + ("kind", kind.as_str()), + ("username", username.unwrap_or("*")), + ] + .into(), + ), + ) + .await?; + tx.commit().await?; + Ok(match username { + Some(username) => format!("Deleted {username}'s draft at {path}"), + None => format!("Deleted {deleted} draft(s) at {path}"), + }) +} + /// For variable-kind drafts with `variable.is_secret == true`, encrypt /// `variable.value` with the workspace crypt key and mark it /// `$encrypted:` so the secret never persists in plaintext at rest. @@ -1161,14 +1270,21 @@ pub(crate) async fn require_can_write_path( if authed.is_admin { return Ok(()); } - // Operators are read-only and never WRITE drafts, except a flow draft where the workspace - // granted the builder right: the kind has to be checked, or the right would open drafts of - // kinds it says nothing about. Read access is deliberately asymmetric: + // Operators are read-only and never WRITE drafts, except of a kind the workspace granted them + // the matching builder right for: a flows-only workspace must not get raw-app drafts through + // here. Read access is deliberately asymmetric: // `require_can_read_path` has no operator block, so an operator can still READ a draft they // can read via `/drafts/get`, mirroring their read access to deployed content. Intended. if authed.is_operator { - let granted = - matches!(kind, UserDraftItemKind::Flow) && operator_can_build_flows(db, w_id).await?; + let allowed = match kind { + UserDraftItemKind::Flow => Some(BuilderKind::Flows), + UserDraftItemKind::RawApp => Some(BuilderKind::Apps), + _ => None, + }; + let granted = match allowed { + Some(kind) => operator_builder_rights(db, w_id).await?.has(kind), + None => false, + }; if !granted { return Err(Error::PermissionDenied( "operators cannot save drafts".to_string(), diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index eb8986a986..0d6f20cdfa 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -263,6 +263,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, + pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } #[derive(Serialize)] @@ -278,7 +282,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } pub async fn load_file_metadata_internal( @@ -308,6 +315,7 @@ mod app_s3_display_stubs { pub async fn load_table_count_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadCountQuery, @@ -320,6 +328,7 @@ mod app_s3_display_stubs { pub async fn load_preview_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadPreviewQuery, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index b08b083695..2b93c6daeb 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -55,7 +55,8 @@ use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; use windmill_common::workspaces::{ - check_operator_can_build_flows, check_user_against_rule, ProtectionRuleKind, RuleCheckResult, + check_operator_can_build, check_user_against_rule, BuilderKind, ProtectionRuleKind, + RuleCheckResult, }; use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "instance_smtp"))] @@ -378,6 +379,10 @@ pub fn workspaced_service() -> Router { "/wac_approval_urls/{job_id}/{step_key}", get(get_wac_approval_urls).layer(cors.clone()), ) + .route( + "/worker_is_draining/{id}", + get(worker_is_draining).layer(cors.clone()), + ) .route( "/result_by_id/{job_id}/{node_id}", get(get_result_by_id).layer(cors.clone()), @@ -613,6 +618,53 @@ async fn get_root_job( Ok(Json(res)) } +/// Whether the worker running this job has received its shutdown signal. A draining worker +/// never interrupts its job, so a long-running script polls this to exit on its own terms. +async fn worker_is_draining( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> windmill_common::error::JsonResult { + // A job always may ask about itself. Anyone else must be able to see the job: the + // caller's row policies on v2_job decide, narrowed by the token's tag filter. + let draining = if authed.job_id == Some(id) { + sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job_queue q JOIN worker_ping wp ON wp.worker = q.worker + WHERE q.id = $1 AND q.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL", + id, + w_id + ) + .fetch_optional(&db) + .await? + } else { + let tags = get_scope_tags(&authed) + .map(|tags| tags.into_iter().map(str::to_string).collect::>()); + let mut tx = user_db.begin(&authed).await?; + let draining = sqlx::query_scalar!( + "SELECT wp.draining FROM v2_job j + JOIN v2_job_queue q ON q.id = j.id + JOIN worker_ping wp ON wp.worker = q.worker + WHERE j.id = $1 AND j.workspace_id = $2 AND q.running AND q.started_at IS NOT NULL + AND ($3::text[] IS NULL OR j.tag = ANY($3))", + id, + w_id, + tags.as_deref() + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + draining + } + .unwrap_or(false); + windmill_common::feature_usage::log_feature_usage( + "worker_draining", + "check", + if draining { "draining" } else { "not_draining" }, + ); + Ok(Json(draining)) +} + async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { let root_job = sqlx::query_scalar!( r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, @@ -1959,9 +2011,11 @@ async fn require_job_within_read_scope( } } // `scope_kind` is the runnable kind a `jobs:run::` scope can name, or - // NULL for a job no such scope reaches directly (previews, dependency jobs, - // flow-inlined scripts) — those are still readable as a step of a matching flow, - // through their ancestors. A `singlestepflow` wraps either a script or a flow, so it + // NULL for a job no such scope reaches directly (previews, flow-inlined scripts) — + // those are still readable as a step of a matching flow, through their ancestors. + // A dependency job's path is the one its `jobs:run:dependencies:` check + // admitted, but it is filed under `dependencies`, so it never passes for a run of + // the deployed runnable at that path. A `singlestepflow` wraps either a script or a flow, so it // projects onto the wrapped runnable the same way the batch-rerun query does. An agent // run is a preview of the one-step flow `agent_runs::agent_step_flow` builds, filed under the // agent's path (`.chat` for a chat turn); the editor's own runs of it look the same, @@ -1980,6 +2034,7 @@ async fn require_job_within_read_scope( WHEN a.agent THEN 'agents' WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts' WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows' + WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies' WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN CASE WHEN COALESCE( (SELECT m->'value'->>'type' @@ -4577,14 +4632,8 @@ async fn cancel_selection( } else { ids }; - let Json(mut w_cancelled) = cancel_jobs( - ids, - &db, - &authed, - workspace_id.as_str(), - force_cancel, - ) - .await?; + let Json(mut w_cancelled) = + cancel_jobs(ids, &db, &authed, workspace_id.as_str(), force_cancel).await?; cancelled.append(&mut w_cancelled); } @@ -7775,6 +7824,7 @@ pub async fn run_workflow_as_code( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, mut tx) = push( &db, @@ -7803,7 +7853,7 @@ pub async fn run_workflow_as_code( None, push_authed.as_ref(), false, - None, + end_user_email, None, None, scope_ceiling.as_deref(), @@ -8090,6 +8140,7 @@ pub async fn run_wait_result_job_by_path_get( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -8118,7 +8169,7 @@ pub async fn run_wait_result_job_by_path_get( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -8240,6 +8291,7 @@ pub async fn run_wait_result_script_by_path_internal( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -8268,7 +8320,7 @@ pub async fn run_wait_result_script_by_path_internal( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -8358,6 +8410,7 @@ pub async fn run_wait_result_script_by_hash( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -8400,7 +8453,7 @@ pub async fn run_wait_result_script_by_hash( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -8541,6 +8594,11 @@ pub async fn stream_job( run_query: RunJobQuery, is_get: bool, ) -> error::Result { + if run_query.retry.is_some() { + return Err(error::Error::BadRequest( + "retry is only supported when running a script asynchronously".to_string(), + )); + } let args = if is_get { let payload_as_args = run_query.payload_as_args()?; @@ -8924,6 +8982,7 @@ async fn run_preview_script( check_tag_available_for_workspace(&db, &w_id, &tag, &push_args, &authed).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let (uuid, tx) = push( &db, tx, @@ -8971,7 +9030,7 @@ async fn run_preview_script( None, Some(&authed.clone().into()), false, - None, + end_user_email, authed.trigger_or_fallback(None), None, scope_ceiling.as_deref(), @@ -9010,6 +9069,7 @@ async fn run_inline_preview_script( if let Some(job_id) = job_id { register_potential_assets_on_inline_execution(job_id, &w_id, &preview); } + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let utils = get_worker_internal_server_inline_utils()?; let result = utils.run_inline_preview_script.as_ref()(RunInlinePreviewScriptFnParams { content: preview.content, @@ -9020,6 +9080,7 @@ async fn run_inline_preview_script( created_by: authed.display_username().to_string(), permissioned_as: username_to_permissioned_as(&authed.username), permissioned_as_email: authed.email.clone(), + end_user_email, lang: preview.language, job_dir: "".to_string(), worker_name: "".to_string(), @@ -9159,6 +9220,7 @@ async fn run_inline_script_inner( .to_string(), )); } + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let utils = get_worker_internal_server_inline_utils()?; let authed_owned: windmill_common::db::Authed = authed.clone().into(); let result = utils.run_inline_script.as_ref()(RunInlineScriptFnParams { @@ -9170,6 +9232,7 @@ async fn run_inline_script_inner( created_by: authed.display_username().to_string(), permissioned_as: username_to_permissioned_as(&authed.username), permissioned_as_email: authed.email.clone(), + end_user_email, job_dir: "".to_string(), worker_name: "".to_string(), worker_dir: "".to_string(), @@ -9346,6 +9409,7 @@ async fn run_bundle_preview_script( // tracing::info!("is_tar 1: {is_tar}"); // hmap.insert("") let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let (uuid, ntx) = push( &db, ltx, @@ -9389,7 +9453,7 @@ async fn run_bundle_preview_script( None, Some(&authed.clone().into()), false, - None, + end_user_email, authed.trigger_or_fallback(None), None, scope_ceiling.as_deref(), @@ -9469,13 +9533,23 @@ pub struct RunDependenciesResponse { pub dependencies: String, } +/// The path comes from the request body: one carrying scope syntax (`,`, `:`, `*`) could +/// put a granted path beside an ungranted one, so it needs a grant covering every path. +fn check_dependencies_scope(authed: &ApiAuthed, path: &str) -> error::Result<()> { + if windmill_common::auth::is_scope_literal_path(path) { + check_scopes(authed, || format!("jobs:run:dependencies:{path}")) + } else { + check_scopes(authed, || "jobs:run:dependencies".to_string()) + } +} + async fn push_dependencies_job( authed: &ApiAuthed, db: &DB, w_id: &str, req: RunDependenciesRequest, ) -> error::Result { - check_scopes(authed, || format!("jobs:run"))?; + check_dependencies_scope(authed, &req.entrypoint)?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run dependencies jobs for security reasons".to_string(), @@ -9598,8 +9672,15 @@ async fn push_flow_dependencies_job( w_id: &str, req: RunFlowDependenciesRequest, ) -> error::Result { - check_scopes(authed, || format!("jobs:run"))?; - check_operator_can_build_flows(db, w_id, authed.is_operator, "run dependencies jobs").await?; + check_dependencies_scope(authed, &req.path)?; + check_operator_can_build( + db, + w_id, + authed.is_operator, + BuilderKind::Flows, + "run dependencies jobs", + ) + .await?; // The dependency job locks whatever inline code this request carries, on a worker. A // composition-only flow has none, so validating here costs a builder nothing and keeps the // lock step from becoming the way to run code the write path refuses. @@ -10014,7 +10095,14 @@ async fn run_preview_flow_job( Query(run_query): Query, Json(raw_flow): Json, ) -> error::Result<(StatusCode, String)> { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "run preview jobs").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "run preview jobs", + ) + .await?; // Flow preview runs an arbitrary, request-supplied flow definition; require the broad // jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script. check_scopes(&authed, || format!("jobs:run"))?; @@ -10069,6 +10157,7 @@ async fn run_preview_flow_job( .await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let (uuid, mut tx) = push( &db, tx, @@ -10100,7 +10189,7 @@ async fn run_preview_flow_job( None, Some(&authed.clone().into()), false, - None, + end_user_email, authed.trigger_or_fallback(None), None, scope_ceiling.as_deref(), @@ -10483,26 +10572,30 @@ pub async fn run_job_by_hash_inner( ) }; + let job_payload = JobPayload::ScriptHash { + hash: ScriptHash(hash), + path: path, + concurrency_settings, + debouncing_settings, + cache_ttl, + cache_ignore_s3_path, + language, + dedicated_worker, + priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + labels, + job_token_scopes, + }; + let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; + + let end_user_email = run_end_user_email(&db, &w_id, &authed, trigger.as_ref()).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, tx, &w_id, - JobPayload::ScriptHash { - hash: ScriptHash(hash), - path: path, - concurrency_settings, - debouncing_settings, - cache_ttl, - cache_ignore_s3_path, - language, - dedicated_worker, - priority, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - labels, - job_token_scopes, - }, + job_payload, push_args, authed.display_username(), email, @@ -10525,7 +10618,7 @@ pub async fn run_job_by_hash_inner( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, scope_ceiling.as_deref(), diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 1cd2b4101c..92291bd1ea 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -149,6 +149,8 @@ pub mod oauth2_oss; #[cfg(feature = "private")] pub mod oidc_ee; mod oidc_oss; +#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] +pub mod oidc_signer_tools_ee; mod path_autocomplete; mod raw_apps; mod resources; diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index 334ee340a8..a620807902 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -8,7 +8,9 @@ use sqlx::{PgConnection, Postgres}; use tokio::task::JoinHandle; -use windmill_common::{db::DB, error::Error}; +use windmill_common::{ + db::DB, error::Error, global_settings::SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, +}; use crate::db::CustomMigrator; use sqlx::migrate::Migrate; @@ -160,7 +162,11 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { return Ok(()); } - for step in [AUDIT_OPERATION_INDEX, RETIRE_LEGACY_AUDIT] { + for step in [ + RETIRE_LEGACY_HUB_SYNC, + AUDIT_OPERATION_INDEX, + RETIRE_LEGACY_AUDIT, + ] { if background_migration_done(&mut conn, step).await? { continue; } @@ -171,6 +177,7 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { let run = match step { AUDIT_OPERATION_INDEX => create_audit_operation_index(&mut conn).await, RETIRE_LEGACY_AUDIT => retire_legacy_audit_table(&mut conn).await, + RETIRE_LEGACY_HUB_SYNC => retire_legacy_hub_sync(&mut conn).await, _ => unreachable!("background migration {step} has no step function"), }; match run { @@ -192,6 +199,7 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { const AUDIT_OPERATION_INDEX: &str = "audit_partitioned_workspace_operation_index"; const RETIRE_LEGACY_AUDIT: &str = "retire_legacy_audit_table"; +const RETIRE_LEGACY_HUB_SYNC: &str = "retire_legacy_hub_sync_script"; // Short on purpose: a statement waiting for its lock is also a wait for every audit insert // queued behind it. @@ -371,6 +379,70 @@ async fn retire_legacy_audit_table(conn: &mut PgConnection) -> Result<(), Error> Ok(()) } +/// The hash the migrations gave `u/admin/hub_sync`; they rewrote its content in place. +const LEGACY_HUB_SYNC_HASH: i64 = -28028598712388162; + +/// `u/admin/hub_sync` and the schedules running it synced resource types before servers did +/// it themselves (`windmill_common::hub_resource_types`). Retired only while the script is +/// still the version Windmill created: a copy someone edited is theirs to keep running. +async fn retire_legacy_hub_sync(conn: &mut PgConnection) -> Result<(), Error> { + let mut tx = conn.begin().await?; + let latest: Option = sqlx::query_scalar( + "SELECT hash FROM script + WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync' AND archived = false + ORDER BY created_at DESC LIMIT 1", + ) + .fetch_optional(&mut *tx) + .await?; + if latest != Some(LEGACY_HUB_SYNC_HASH) { + return Ok(()); + } + + let schedules: Vec<(String, bool, serde_json::Value)> = sqlx::query_as( + "SELECT path, enabled, jsonb_build_object('row', to_jsonb(s)) FROM schedule s + WHERE workspace_id = 'admins' AND script_path = 'u/admin/hub_sync' AND NOT is_flow", + ) + .fetch_all(&mut *tx) + .await?; + // An enabled schedule hands over to the server's daily sync in the transaction that deletes + // it, so the instance never has both syncs running, nor neither. + if schedules.iter().any(|(_, enabled, _)| *enabled) { + let inserted = sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, 'true'::jsonb) + ON CONFLICT (name) DO NOTHING", + ) + .bind(SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + tracing::info!( + "u/admin/hub_sync ran on an enabled schedule: turned on {SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING}" + ); + } + } + for (path, _, row) in schedules { + windmill_queue::schedule::clear_schedule(&mut tx, &path, "admins").await?; + sqlx::query("DELETE FROM schedule WHERE workspace_id = 'admins' AND path = $1") + .bind(&path) + .execute(&mut *tx) + .await?; + windmill_common::trashbin::move_to_trash( + &mut *tx, "admins", "schedule", &path, row, "system", + ) + .await?; + tracing::info!("Deleted schedule {path}, which ran u/admin/hub_sync"); + } + sqlx::query( + "UPDATE script SET archived = true WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync'", + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + tracing::info!("Archived u/admin/hub_sync: servers sync resource types themselves"); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -412,4 +484,80 @@ mod tests { assert_eq!(seen, 2); Ok(()) } + + async fn insert_legacy_hub_sync_schedule(db: &DB, enabled: bool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, email, permissioned_as, schedule, timezone, script_path, is_flow, enabled) + VALUES ('admins', 'g/all/hub_sync', 'admin', 'admin@windmill.dev', 'u/admin', '0 0 0 * * *', 'Etc/UTC', 'u/admin/hub_sync', false, $1)", + ) + .bind(enabled) + .execute(db) + .await?; + Ok(()) + } + + async fn schedules_and_live_versions(db: &DB) -> anyhow::Result<(i64, i64)> { + Ok(sqlx::query_as( + "SELECT (SELECT count(*) FROM schedule WHERE workspace_id = 'admins'), + (SELECT count(*) FROM script WHERE path = 'u/admin/hub_sync' AND NOT archived)", + ) + .fetch_one(db) + .await?) + } + + async fn daily_sync_setting(db: &DB) -> anyhow::Result> { + Ok( + sqlx::query_scalar("SELECT value FROM global_settings WHERE name = $1") + .bind(SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING) + .fetch_optional(db) + .await?, + ) + } + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_hub_sync_spares_an_edited_script(db: DB) -> anyhow::Result<()> { + insert_legacy_hub_sync_schedule(&db, true).await?; + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, created_by, language, summary, description, schema, lock, created_at) + SELECT workspace_id, 42, path, content || '// edited', 'admin', language, summary, description, schema, lock, now() + interval '1 minute' + FROM script WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync'", + ) + .execute(&db) + .await?; + + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (1, 2)); + assert_eq!(daily_sync_setting(&db).await?, None); + + // Unedited, the enabled schedule hands over to the server's daily sync. + sqlx::query("DELETE FROM script WHERE hash = 42") + .execute(&db) + .await?; + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (0, 0)); + assert_eq!( + daily_sync_setting(&db).await?, + Some(serde_json::json!(true)) + ); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_hub_sync_leaves_the_daily_sync_off_without_an_enabled_schedule( + db: DB, + ) -> anyhow::Result<()> { + for disabled_schedule in [true, false] { + sqlx::query("UPDATE script SET archived = false WHERE path = 'u/admin/hub_sync'") + .execute(&db) + .await?; + if disabled_schedule { + insert_legacy_hub_sync_schedule(&db, false).await?; + } + + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (0, 0)); + assert_eq!(daily_sync_setting(&db).await?, None); + } + Ok(()) + } } diff --git a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs index 1962710578..73b1fd1792 100644 --- a/backend/windmill-api/src/mcp/auto_generated_endpoints.rs +++ b/backend/windmill-api/src/mcp/auto_generated_endpoints.rs @@ -2024,7 +2024,7 @@ You should get the schema of the script or flow before creating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2037,7 +2037,7 @@ You should get the schema of the script or flow before creating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", @@ -2231,7 +2231,7 @@ You should get the schema of the script or flow before updating the schedule to }, "exponential": { "type": "object", - "description": "Retry with exponential backoff (delay doubles each time)", + "description": "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n", "properties": { "attempts": { "type": "integer", @@ -2244,7 +2244,7 @@ You should get the schema of the script or flow before updating the schedule to "seconds": { "type": "integer", "minimum": 1, - "description": "Initial delay in seconds" + "description": "Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too" }, "random_factor": { "type": "integer", diff --git a/backend/windmill-api/src/mcp_tools.rs b/backend/windmill-api/src/mcp_tools.rs index e1931873a9..1e07f374de 100644 --- a/backend/windmill-api/src/mcp_tools.rs +++ b/backend/windmill-api/src/mcp_tools.rs @@ -23,14 +23,12 @@ async fn with_deadline( what: &str, fut: impl std::future::Future>, ) -> Result { - tokio::time::timeout(MCP_DEADLINE, fut) - .await - .map_err(|_| { - Error::ExecutionErr(format!( - "MCP server did not answer within {}s ({what})", - MCP_DEADLINE.as_secs() - )) - })? + tokio::time::timeout(MCP_DEADLINE, fut).await.map_err(|_| { + Error::ExecutionErr(format!( + "MCP server did not answer within {}s ({what})", + MCP_DEADLINE.as_secs() + )) + })? } /// Connect to the MCP server described by the `mcp` resource at `path`. @@ -65,8 +63,9 @@ async fn connect_mcp_client( let resource_value = not_found_if_none(resource_value_o, "Resource", path)? .ok_or_else(|| Error::BadRequest(format!("Empty resource value for {}", path)))?; - let mcp_resource = serde_json::from_str::(resource_value.0.get()) - .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; + let mut mcp_resource = + serde_json::from_str::(resource_value.0.get()) + .map_err(|e| Error::BadRequest(format!("Failed to parse MCP resource: {}", e)))?; #[cfg(feature = "oauth2")] { @@ -126,6 +125,15 @@ async fn connect_mcp_client( None }; + // Header secrets go through the same permissioned path as the token. + for value in mcp_resource.headers.iter_mut().flatten().map(|(_, v)| v) { + if let Some(var_path) = value.strip_prefix("$var:") { + check_scopes(authed, || format!("variables:read:{}", var_path))?; + let db_authed = DbWithOptAuthed::from_authed(authed, db.clone(), Some(user_db.clone())); + *value = get_value_internal(&db_authed, w_id, var_path, false).await?; + } + } + windmill_mcp::McpClient::from_resource(mcp_resource, token) .await .map_err(|e| Error::ExecutionErr(format!("Failed to connect to MCP server: {}", e))) diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index 846089630b..7e64b48a1a 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -34,7 +34,7 @@ use std::collections::HashMap; use windmill_common::{ db::UserDB, error::{Error, JsonResult}, - workspaces::operator_can_build_flows, + workspaces::{operator_builder_rights, OperatorBuilderRights}, }; use windmill_types::scripts::ScriptHash; use windmill_types::user_drafts::DraftUserRef; @@ -574,9 +574,14 @@ async fn list_runnables( // Draft-only rows are the caller's own work in progress: never archived, so they // have no place in the archived view, and carrying no labels of their own they are // out of scope of a label filter (as in the per-kind endpoints). Operators don't - // see other people's drafts and have none of their own, except a builder's flows. + // see other people's drafts and have none of their own, except of what they may build. + let builder = if authed.is_operator { + operator_builder_rights(&db, &w_id).await? + } else { + OperatorBuilderRights::default() + }; let include_drafts = q.include_draft_only.unwrap_or(false) - && (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?) + && (!authed.is_operator || builder.any()) && !show_archived && q.label.as_ref().filter(|s| !s.is_empty()).is_none(); let draft_extras_for = |kind: &str| -> Vec { @@ -662,7 +667,10 @@ async fn list_runnables( keyset: Option<&str>, limit: Option| -> Option { - if !include_drafts || !kinds.contains(&kind) || (authed.is_operator && kind != "flow") { + if !include_drafts + || !kinds.contains(&kind) + || (authed.is_operator && !operator_drafts_kind(builder, kind)) + { return None; } // `fav` is ignored: with no favorite join there is nothing to filter on, and the @@ -990,6 +998,15 @@ async fn count_runnables_by_owner( Ok(Json(RunnableCountsResponse { counts })) } +/// Whether an operator with `builder` rights can have drafts of a `/list` kind. +fn operator_drafts_kind(builder: OperatorBuilderRights, kind: &str) -> bool { + match kind { + "flow" => builder.flows, + "app" => builder.apps, + _ => false, + } +} + /// Adds the caller's draft-only rows to `counts`, in the same shape `/list` /// returns them so a badge never disagrees with the rows behind it. /// @@ -1010,13 +1027,21 @@ async fn add_draft_counts( if !q.include_draft_only.unwrap_or(false) { return Ok(()); } - // An operator has no drafts of their own, except a builder's flows. + // An operator has no drafts of their own, except of what they may build. + let builder_kinds: Vec<&str>; let kinds: &[&str] = if !authed.is_operator { kinds - } else if kinds.contains(&"flow") && operator_can_build_flows(db, w_id).await? { - &["flow"] } else { - return Ok(()); + let builder = operator_builder_rights(db, w_id).await?; + builder_kinds = kinds + .iter() + .copied() + .filter(|k| operator_drafts_kind(builder, k)) + .collect(); + if builder_kinds.is_empty() { + return Ok(()); + } + &builder_kinds }; // $1 = workspace, $2 = the caller's email. let mut binds: Vec = vec![]; diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index b7fbe96404..ab22f2d15b 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -215,6 +215,11 @@ lazy_static! { label: "Run flows".to_string(), requires_resource_path: true, }, + ScopeOption { + value: "jobs:run:dependencies".to_string(), + label: "Generate dependency locks".to_string(), + requires_resource_path: true, + }, ], }]; diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 39a213c860..5b26cf57b8 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -198,6 +198,7 @@ async fn set_password( Extension(db): Extension, Extension(argon2): Extension>>, authed: ApiAuthed, + Tokened { token }: Tokened, OptJobAuthed { job_id, .. }: OptJobAuthed, Json(ep): Json, ) -> Result { @@ -206,7 +207,15 @@ async fn set_password( // (GHSA-hfh4-cx4h-3fcr). forbid_elevated_job_token(&db, &authed.email, job_id).await?; let email = authed.email.clone(); - crate::users_oss::set_password(db, argon2, authed, &email, ep).await + let res = crate::users_oss::set_password(db.clone(), argon2, authed, &email, ep).await?; + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session' AND token_hash != $2", + &email, + windmill_common::auth::hash_token(&token) + ) + .execute(&db) + .await?; + Ok(res) } async fn set_password_of_user( @@ -219,7 +228,14 @@ async fn set_password_of_user( ) -> Result { require_super_admin(&db, &authed).await?; forbid_superadmin_job_token(&db, &authed.email, job_id).await?; - crate::users_oss::set_password(db, argon2, authed, &email, ep).await + let res = crate::users_oss::set_password(db.clone(), argon2, authed, &email, ep).await?; + sqlx::query!( + "DELETE FROM token WHERE email = $1 AND label = 'session'", + &email + ) + .execute(&db) + .await?; + Ok(res) } #[derive(Deserialize)] diff --git a/backend/windmill-audit/src/instance_config.rs b/backend/windmill-audit/src/instance_config.rs new file mode 100644 index 0000000000..977e0fe6c6 --- /dev/null +++ b/backend/windmill-audit/src/instance_config.rs @@ -0,0 +1,148 @@ +//! Audit entries for writes to instance settings and worker group configs, whichever way +//! they arrive: an API request, or a declarative sync that no user request stands behind. + +use std::collections::HashMap; + +use windmill_common::{ + audit::AuditAuthor, error::Result, instance_config::ConfigWriteObserver, + users::SUPERADMIN_SYNC_EMAIL, DB, +}; + +use crate::{ + audit_oss::{audit_log, AuditAuthorable}, + ActionKind, +}; + +/// The declarative writer behind an entry, recorded as its `source` parameter. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeclarativeSource { + /// `windmill sync-config ` + SyncConfig, + /// The Kubernetes operator's ConfigMap sync. + Operator, +} + +impl DeclarativeSource { + pub fn as_str(&self) -> &'static str { + match self { + DeclarativeSource::SyncConfig => "sync-config", + DeclarativeSource::Operator => "operator", + } + } +} + +/// Who a declarative sync's writes are attributed to. It runs on direct database +/// credentials, so there is no user to name. +pub fn declarative_sync_author() -> AuditAuthor { + AuditAuthor { + username: "instance_config_sync".to_string(), + email: SUPERADMIN_SYNC_EMAIL.to_string(), + username_override: None, + token_prefix: None, + } +} + +fn parameters<'a>( + value: Option<&'a str>, + source: Option, +) -> Option> { + let parameters: HashMap<&str, &str> = value + .map(|v| ("value", v)) + .into_iter() + .chain(source.map(|s| ("source", s.as_str()))) + .collect(); + (!parameters.is_empty()).then_some(parameters) +} + +/// A setting is recorded by name, with its value only when that is a boolean or a number. +/// Any other shape can carry a credential (a registry URL, a webhook, a header map), and no +/// list of secret-bearing settings stays complete as settings are added. +/// +/// AUTHORIZATION: records a write, checks nothing. The caller must have established that +/// `author` may change instance settings before performing the write this records. +pub async fn audit_setting_write( + db: &DB, + author: &impl AuditAuthorable, + key: &str, + value: Option<&serde_json::Value>, + source: Option, +) -> Result<()> { + let deleted = match value { + None | Some(serde_json::Value::Null) => true, + Some(serde_json::Value::String(s)) => s.trim().is_empty(), + Some(_) => false, + }; + let recorded = match value { + Some(v @ (serde_json::Value::Bool(_) | serde_json::Value::Number(_))) => { + Some(v.to_string()) + } + _ => None, + }; + audit_log( + db, + author, + if deleted { + "global_settings.delete" + } else { + "global_settings.update" + }, + if deleted { + ActionKind::Delete + } else { + ActionKind::Update + }, + "global", + Some(key), + parameters(recorded.as_deref(), source), + ) + .await +} + +/// A worker group config is recorded by name only: its env vars and init script are free +/// text that can carry a credential. +/// +/// AUTHORIZATION: records a write, checks nothing. The caller must have established that +/// `author` may change worker group configs before performing the write this records. +pub async fn audit_worker_config_write( + db: &DB, + author: &impl AuditAuthorable, + group: &str, + deleted: bool, + source: Option, +) -> Result<()> { + let (operation, action_kind) = if deleted { + ("worker_config.delete", ActionKind::Delete) + } else { + ("worker_config.update", ActionKind::Update) + }; + audit_log( + db, + author, + operation, + action_kind, + "global", + Some(&format!("worker__{group}")), + parameters(None, source), + ) + .await +} + +/// Audits each write of an instance config apply as `author`'s. +/// +/// AUTHORIZATION: as [`audit_setting_write`], for every write it is told of. +pub struct InstanceConfigAuditor<'a, A> { + pub db: &'a DB, + pub author: &'a A, + /// `None` for a write made through the API. + pub source: Option, +} + +impl ConfigWriteObserver for InstanceConfigAuditor<'_, A> { + async fn setting_written(&self, key: &str, value: Option<&serde_json::Value>) -> Result<()> { + audit_setting_write(self.db, self.author, key, value, self.source).await + } + + async fn worker_config_written(&self, group: &str, deleted: bool) -> Result<()> { + audit_worker_config_write(self.db, self.author, group, deleted, self.source).await + } +} diff --git a/backend/windmill-audit/src/lib.rs b/backend/windmill-audit/src/lib.rs index f1443fdb18..e9c71f1336 100644 --- a/backend/windmill-audit/src/lib.rs +++ b/backend/windmill-audit/src/lib.rs @@ -4,6 +4,7 @@ use sqlx::FromRow; #[cfg(feature = "private")] pub mod audit_ee; pub mod audit_oss; +pub mod instance_config; #[derive(sqlx::Type, Serialize, Deserialize, Debug)] #[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")] diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 4d43206ebc..e447baf563 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -8,7 +8,7 @@ edition.workspace = true default = [] enterprise = ["dep:aws-config"] instance_config_schema = ["dep:schemars"] -local_reports = ["dep:rsa", "dep:aes-gcm"] +local_reports = ["dep:rsa"] private = ["dep:aws-sdk-rds", "dep:aws-sdk-secretsmanager", "dep:aws-config"] jemalloc = ["dep:tikv-jemalloc-ctl"] tantivy = [] @@ -22,7 +22,7 @@ smtp = ["dep:mail-send", "dep:rustls", "dep:tokio-rustls", "dep:rustls-native-ce scoped_cache = [] cloud = [] dev_override = [] -openidconnect = ["dep:openidconnect", "dep:rsa"] +openidconnect = ["dep:openidconnect", "dep:rsa", "dep:p256"] python = ["dep:windmill-parser-py"] [lib] @@ -116,9 +116,11 @@ pep440_rs.workspace = true systemstat.workspace = true size.workspace = true rsa = { workspace = true, optional = true } +p256 = { workspace = true, optional = true } spki = { workspace = true } pkcs1 = { workspace = true } -aes-gcm = { workspace = true, optional = true } +aes-gcm.workspace = true +hkdf.workspace = true semver.workspace = true croner.workspace = true diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index 36b2d9b704..1686c20d2b 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -18,6 +18,59 @@ lazy_static::lazy_static! { pub static ref APP_WORKSPACED_ROUTE: AtomicBool = AtomicBool::new(false); } +/// Whether the app value carries an `inlineScript` anywhere. +/// +/// Deliberately not built on [`traverse_app_inline_scripts`], which only reports a script whose +/// `language` parses and stops descending as soon as it sees an `inlineScript` key. That is right +/// for locking (nothing to lock without a language) and wrong for an authorization check, where +/// the author picks the fields: this refuses the key itself, whatever it contains. +pub fn app_value_has_inline_script(value: &Value) -> bool { + match value { + Value::Object(object) => { + object.contains_key("inlineScript") || object.values().any(app_value_has_inline_script) + } + Value::Array(array) => array.iter().any(app_value_has_inline_script), + _ => false, + } +} + +/// Every workspace runnable the app value points a component at, as `(is_flow, path)`. +/// +/// This is what the deployed bundle actually asks `execute_component` to run: it resolves a +/// `runnable_id` against the stored `runnables` and sends `{runType}/{path}`. The policy's +/// triggerables are a separate surface, so both have to be authorized. Anything not plain text +/// is refused, never skipped: the editor compares `type` loosely (`["path"] == "path"`) and +/// interpolates `path` (`["u/x"]` → `u/x`), so a skipped entry still runs. +pub fn app_value_runnable_paths(value: &Value) -> error::Result> { + let bad = |msg: String| error::Error::BadRequest(msg); + let runnables = match value.get("runnables") { + None | Some(Value::Null) => return Ok(vec![]), + Some(Value::Object(runnables)) => runnables, + Some(_) => return Err(bad("App runnables must be an object".to_string())), + }; + let mut out = Vec::new(); + for (id, runnable) in runnables { + let Value::Object(runnable) = runnable else { + return Err(bad(format!("Runnable {id} must be an object"))); + }; + match runnable.get("type") { + Some(Value::String(t)) if t == "runnableByPath" || t == "path" => {} + None | Some(Value::String(_)) => continue, + Some(_) => return Err(bad(format!("Runnable {id} has a non-text type"))), + } + let Some(Value::String(path)) = runnable.get("path") else { + return Err(bad(format!("Runnable {id} must name its path as text"))); + }; + let is_flow = match runnable.get("runType") { + Some(Value::String(t)) if t == "flow" => true, + Some(Value::String(t)) if t == "script" || t == "hubscript" => false, + _ => return Err(bad(format!("Unsupported runType for runnable {path}"))), + }; + out.push((is_flow, path.clone())); + } + Ok(out) +} + /// Traverse FlowValue while invoking provided by caller callback on leafs // #[async_recursion::async_recursion(?Send)] pub fn traverse_app_inline_scripts< diff --git a/backend/windmill-common/src/datatable_roles.rs b/backend/windmill-common/src/datatable_roles.rs index f355e733cb..522327040d 100644 --- a/backend/windmill-common/src/datatable_roles.rs +++ b/backend/windmill-common/src/datatable_roles.rs @@ -72,6 +72,11 @@ impl DatatableRoleCluster { /// created, renamed or dropped. pub const ADMIN_DATATABLE_ROLE: &str = "admin"; +/// Stands in a data table's `default_role` for "none": a connection that names no role then holds +/// its viewer's roles only, and is refused without any. `$` is outside both the role-name charset +/// and the catalog ids, and a server predating it refuses it as a role the data table lacks. +pub const NO_DEFAULT_DATATABLE_ROLE: &str = "$none"; + /// The login the admin connection uses, and the role every created role is granted to — that /// membership is what later lets it `ALTER ... OWNER TO` a role and drop it. pub const CUSTOM_INSTANCE_USER: &str = "custom_instance_user"; @@ -121,8 +126,13 @@ fn is_reserved_role_name(name: &str) -> bool { || lower.starts_with("pg_") || lower.starts_with("windmill_") || lower.starts_with("custom_instance_") + || lower.starts_with(ROLE_SET_LOGIN_PREFIX) } +/// Prefix of the logins Windmill creates for a connection made as several roles at once. Reserved, +/// so no data table role can be created under, renamed to or taken over at such a name. +pub const ROLE_SET_LOGIN_PREFIX: &str = "wm_set_"; + /// The charset is what makes every downstream interpolation safe: the name reaches Postgres as a /// quoted identifier, a `-- role ` annotation, and a `?role=` query parameter. pub fn validate_role_name(name: &str) -> Result<()> { @@ -331,6 +341,12 @@ pub async fn converge_connect_grants_with( /// `CREATE ROLE LOGIN PASSWORD ...; GRANT TO custom_instance_user` on `cluster`. No /// privileges beyond that — an admin grants them through SQL or the ACL editor. /// +/// A role of that name that already exists is refused with a 409, unless `take_over`: then its +/// password is replaced and it is managed from here on like one created here. A role holding +/// cluster-wide privileges, or inheriting another role's, is refused either way, since every tenant +/// would get them. Returns whether an existing role was taken over: undoing a failed take-over +/// must not drop a role that was there before. +/// /// On Windmill's own cluster the DDL runs on `tx`, so it commits with the catalog row. The external /// cluster is another server: the role is created there before `tx` commits, and callers MUST drop /// it again ([`drop_datatable_role`]) if `tx` then fails to commit. @@ -343,8 +359,10 @@ pub async fn create_datatable_role( cluster: DatatableRoleCluster, name: &str, password: &str, -) -> Result<()> { - crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password).await + take_over: bool, +) -> Result { + crate::datatable_roles_oss::create_datatable_role(db, tx, cluster, name, password, take_over) + .await } /// Authorization: alters a cluster-wide Postgres login. Callers MUST restrict this to superadmin @@ -406,6 +424,13 @@ pub async fn drop_datatable_role( crate::datatable_roles_oss::drop_datatable_role(db, tx, cluster, name).await } +/// Drop the logins made for a connection as several roles at once that nobody has connected with +/// for a month, on every cluster. They are made again on demand, so this only bounds how many +/// exist. Safe to call from any server at any time: it serializes on the role catalog lock. +pub async fn sweep_unused_role_set_logins(db: &DB) -> Result<()> { + crate::datatable_roles_oss::sweep_unused_role_set_logins(db).await +} + #[cfg(test)] mod tests { use super::*; diff --git a/backend/windmill-common/src/datatable_roles_oss.rs b/backend/windmill-common/src/datatable_roles_oss.rs index a97a936827..739f08d60b 100644 --- a/backend/windmill-common/src/datatable_roles_oss.rs +++ b/backend/windmill-common/src/datatable_roles_oss.rs @@ -29,7 +29,8 @@ pub(crate) use crate::datatable_roles_ee::{ ensure_instance_db_grant_options_unchecked, forget_datatable_role_everywhere, insert_role_catalog_entry, read_role_catalog, read_role_catalog_tx, registered_instance_databases, rename_datatable_role, resolve_datatable_role_connection, - role_cluster, set_datatable_role_login, update_role_catalog_entry, + role_cluster, set_datatable_role_login, sweep_unused_role_set_logins, + update_role_catalog_entry, }; #[cfg(not(all(feature = "private", feature = "enterprise")))] @@ -125,7 +126,8 @@ mod ce { _cluster: DatatableRoleCluster, _name: &str, _password: &str, - ) -> Result<()> { + _take_over: bool, + ) -> Result { Err(unavailable()) } @@ -234,6 +236,11 @@ mod ce { } } + /// No roles, so no set login to sweep. + pub(crate) async fn sweep_unused_role_set_logins(_db: &DB) -> Result<()> { + Ok(()) + } + pub(crate) async fn forget_datatable_role_everywhere( _tx: &mut Tx<'_>, _role_id: &str, diff --git a/backend/windmill-common/src/external_instance_pg.rs b/backend/windmill-common/src/external_instance_pg.rs index c0b6203e40..72920a1874 100644 --- a/backend/windmill-common/src/external_instance_pg.rs +++ b/backend/windmill-common/src/external_instance_pg.rs @@ -198,7 +198,8 @@ pub async fn external_instance_database_usages<'c>( /// would be a login nothing can drop any more. Allowed on every edition, so a /// downgraded instance can still clear a setting it no longer uses. async fn ensure_external_instance_pg_removable(conn: &mut sqlx::PgConnection) -> Result<()> { - ensure_external_instance_pg_unused(conn, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}")).await + ensure_external_instance_pg_unused(conn, &format!("removing {EXTERNAL_INSTANCE_PG_SETTING}")) + .await } /// Refuse while Windmill has databases or data table roles on the cluster, or a workspace points @@ -394,16 +395,21 @@ pub async fn write_external_instance_pg_setting( } /// [`write_external_instance_pg_setting`] for a settings diff: writes the key if the diff touches -/// it, and takes it out of the diff so the generic apply does not write it again. +/// it, and takes it out of the diff so the generic apply does not write it again. `observer` is +/// told of the write here, since the generic apply no longer sees the key. /// /// Authorization: checks nothing. Callers MUST be superadmin, or the declarative instance config /// sync, which applies what the operator deployed. pub async fn write_external_instance_pg_from_diff( db: &DB, diff: &mut crate::instance_config::SettingsDiff, + observer: &impl crate::instance_config::ConfigWriteObserver, ) -> Result<()> { if let Some(value) = diff.upserts.remove(EXTERNAL_INSTANCE_PG_SETTING) { write_external_instance_pg_setting(db, Some(&value)).await?; + observer + .setting_written(EXTERNAL_INSTANCE_PG_SETTING, Some(&value)) + .await?; } if let Some(i) = diff .deletes @@ -412,6 +418,9 @@ pub async fn write_external_instance_pg_from_diff( { diff.deletes.remove(i); write_external_instance_pg_setting(db, None).await?; + observer + .setting_written(EXTERNAL_INSTANCE_PG_SETTING, None) + .await?; } Ok(()) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index e1a731f3e6..2bc303f5c2 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -158,6 +158,7 @@ pub const SSO_GROUPS_CLAIM_SETTING: &str = "sso_groups_claim"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const DISABLE_HUB_SETTING: &str = "disable_hub"; +pub const SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING: &str = "sync_hub_resource_types_daily"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; @@ -181,6 +182,12 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Base URL the UI shows for the endpoints external clients call (webhooks, HTTP +/// routes, push trigger endpoints, the CLI), instead of the origin the UI is browsed on. +/// For deployments that serve the API on a separate domain or behind a gateway. +/// EE only. Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]: +/// every user is shown URLs built from it. +pub const API_BASE_URL_SETTING: &str = "api_base_url"; /// Instance-wide announcement rendered above every page of the app (maintenance /// windows, incidents). Readable by any authenticated user, unlike most settings: /// the banner exists to be shown to everyone, so it must never hold anything the @@ -313,12 +320,17 @@ pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> { pub struct InstanceUi { pub instance_banner: Option, pub accent_color: Option, + pub api_base_url: Option, } pub async fn get_instance_ui(db: &Pool) -> error::Result { let rows = sqlx::query!( "SELECT name, value FROM global_settings WHERE name = ANY($1)", - &[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str] + &[ + INSTANCE_BANNER_SETTING, + ACCENT_COLOR_SETTING, + API_BASE_URL_SETTING + ] as &[&str] ) .fetch_all(db) .await?; @@ -327,17 +339,22 @@ pub async fn get_instance_ui(db: &Pool) -> error::Result { match row.name.as_str() { INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value), ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value), + API_BASE_URL_SETTING => ui.api_base_url = Some(row.value), _ => {} } } Ok(ui) } -/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. +const SHELL_ACTIVE_CHARS: [char; 16] = [ + '$', '`', '\'', '"', '\\', ';', '&', '|', '<', '>', '(', ')', '{', '}', '*', '!', +]; + +/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] or [`API_BASE_URL_SETTING`] value. /// -/// The receiver path is appended to it verbatim, so anything that doesn't -/// concatenate into a URL GitHub can POST to must be rejected at write time -/// rather than silently producing an unreachable hook: a wrong scheme +/// A path is appended to it verbatim, so anything that doesn't concatenate into a +/// URL an external caller can reach must be rejected at write time rather than +/// silently producing an unreachable hook or endpoint: a wrong scheme /// (`httpss://`), a missing host, embedded whitespace, or a query/fragment /// (appending a path after `?`/`#` keeps it inside the query/fragment). /// @@ -353,7 +370,7 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { url::Url::parse(value).map_err(|e| format!("must be an absolute http(s) URL: {e}"))?; if !url.username().is_empty() || url.password().is_some() { return Err( - "must not embed a username or password: the receiver URL is stored in workspace settings, where it is readable by workspace admins".to_string(), + "must not embed a username or password: the URL is readable by users who are not instance admins".to_string(), ); } if !matches!(url.scheme(), "http" | "https") { @@ -366,13 +383,21 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { } if url.query().is_some() || url.fragment().is_some() { return Err( - "must not include a query string or fragment, since the webhook path is appended to it" + "must not include a query string or fragment, since a path is appended to it" .to_string(), ); } if value.chars().any(char::is_whitespace) { return Err("must not contain whitespace".to_string()); } + // The UI pastes the URL into shell commands users copy, where double quotes + // do not stop `$(...)` or backticks from running. + if value.contains(SHELL_ACTIVE_CHARS) { + return Err( + "must not contain shell metacharacters, since it is shown inside commands users copy" + .to_string(), + ); + } // Matches the sibling `base_url` / `hub_base_url` convention. The receiver // builder trims it anyway, so this is about keeping the stored value canonical // rather than about reachability. @@ -1101,6 +1126,9 @@ mod tests { "https://hooks.example.com#", "https://user:password@hooks.example.com", "https://user@hooks.example.com", + "https://hooks.example.com/$(uname)", + "https://hooks.example.com/`uname`", + "https://hooks.example.com/a;b", ]; for v in accept { assert!( diff --git a/backend/windmill-common/src/hub_resource_types.rs b/backend/windmill-common/src/hub_resource_types.rs new file mode 100644 index 0000000000..88de520094 --- /dev/null +++ b/backend/windmill-common/src/hub_resource_types.rs @@ -0,0 +1,578 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Keeps the `admins` workspace's resource types, which every workspace falls back to, in +//! step with the hub. Servers run [`spawn_daily_sync`] once a superadmin turns it on, +//! superadmins run [`sync`] on demand, and `windmill cache-rt` bakes the public hub's listing +//! into the image for instances that cannot reach a hub. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::{ + error::{Error, Result}, + global_settings::{ + load_value_from_global_settings, DISABLE_HUB_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, + }, + utils::{get_license_id_or_uid, HTTP_CLIENT_PERMISSIVE, HUB_API_SECRET}, + worker::HUB_RT_CACHE_DIR, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, +}; + +const SYNC_INTERVAL: chrono::TimeDelta = chrono::TimeDelta::hours(24); +const RETRY_INTERVAL: chrono::TimeDelta = chrono::TimeDelta::hours(1); +const CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(600); +const SYNC_LOCK_ID: i64 = 737_483_925; +/// `background_task_state.name` of the sync's last attempt and last hub sync. +const SYNC_STATE_TASK: &str = "hub_resource_type_sync"; + +pub fn cache_path() -> String { + format!("{}/resource_types.json", *HUB_RT_CACHE_DIR) +} + +/// A hub resource type with its schema parsed, as synced and as `cache-rt` stores it. +/// +/// `format_extension` and `display_name` are doubly optional: no key (a cache written before +/// the field) leaves the stored value alone, while an explicit null clears it. Plain serde +/// folds both into `None`, hence the wrapping deserializer. +#[derive(Deserialize, Serialize, Clone)] +pub struct HubResourceType { + pub name: String, + pub schema: Option, + pub description: Option, + #[serde( + default, + deserialize_with = "crate::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub format_extension: Option>, + #[serde( + default, + deserialize_with = "crate::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub display_name: Option>, + /// Absent from hubs that do not publish filesets, which leaves the stored flag alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub is_fileset: Option, +} + +/// An entry of the hub's `/resource_types/list`, whose schema is a JSON string. +#[derive(Deserialize)] +struct HubListing { + name: String, + schema: Option, + description: Option, + /// Absent from hubs predating the column, which do not publish file types either, so + /// the hub is authoritative for it whether or not it sends the key. + #[serde(default)] + format_extension: Option, + #[serde(default, deserialize_with = "crate::more_serde::double_option")] + display_name: Option>, + #[serde(default)] + is_fileset: Option, +} + +/// One hub's listing. A type whose schema does not parse is left out. +pub async fn fetch( + hub_base_url: &str, + api_secret: Option<&str>, + uid: Option<&str>, +) -> Result> { + // Permissive, so `ACCEPT_INVALID_CERTS` lets the sync reach a private hub behind a + // self-signed certificate. + let mut request = HTTP_CLIENT_PERMISSIVE + .get(format!("{hub_base_url}/resource_types/list")) + .header("Accept", "application/json"); + if let Some(uid) = uid { + request = request.header("X-uid", uid); + } + if let Some(secret) = api_secret { + request = request.header("X-api-secret", secret); + } + let response = request.send().await.map_err(|e| { + Error::InternalErr(format!("Failed to reach the hub at {hub_base_url}: {e}")) + })?; + let status = response.status(); + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(Error::InternalErr(format!( + "The hub at {hub_base_url} refused the request: check the Private Hub API secret" + ))); + } + if !status.is_success() { + return Err(Error::InternalErr(format!( + "The hub at {hub_base_url} answered {status}" + ))); + } + let listing: Vec = response.json().await.map_err(|e| { + Error::InternalErr(format!( + "Failed to parse the resource types listed by {hub_base_url}: {e}" + )) + })?; + + Ok(listing + .into_iter() + .filter_map(|rt| { + let schema = match rt.schema.as_deref().map(serde_json::from_str).transpose() { + Ok(schema) => schema, + Err(e) => { + tracing::warn!( + "Skipping hub resource type {}: invalid schema: {e}", + rt.name + ); + return None; + } + }; + Some(HubResourceType { + name: rt.name, + schema, + description: rt.description, + format_extension: Some(rt.format_extension), + display_name: rt.display_name, + is_fileset: rt.is_fileset, + }) + }) + .collect()) +} + +/// The listing of the hub this instance is configured with. +async fn fetch_configured_hub(db: &DB) -> Result> { + let hub = (**HUB_BASE_URL.load()).clone(); + let uid = get_license_id_or_uid(db).await.ok(); + if hub == DEFAULT_HUB_BASE_URL { + return fetch(&hub, None, uid.as_deref()).await; + } + let secret = (**HUB_API_SECRET.load()).clone(); + let types = fetch(&hub, secret.as_deref(), uid.as_deref()).await?; + if !types.is_empty() { + return Ok(types); + } + // A private hub that publishes no resource types of its own defers to the public one, + // which must not be sent the private hub's secret. + tracing::info!("The hub at {hub} lists no resource types, syncing the public hub's"); + fetch(DEFAULT_HUB_BASE_URL, None, uid.as_deref()).await +} + +#[derive(PartialEq, Debug)] +struct ResourceTypeRow { + schema: Option, + description: Option, + format_extension: Option, + is_fileset: bool, + display_name: Option, +} + +/// The row a synced type should end up as, given what `admins` already stores for it. +fn target_row(rt: &HubResourceType, stored: Option<&ResourceTypeRow>) -> ResourceTypeRow { + let is_fileset = rt + .is_fileset + .unwrap_or_else(|| stored.is_some_and(|s| s.is_fileset)); + // A fileset is a set of files, so it cannot also be one file. Create and update reject + // the pair; this writer bypasses both. + let format_extension = if is_fileset { + None + } else { + match &rt.format_extension { + Some(extension) => extension.clone(), + None => stored.and_then(|s| s.format_extension.clone()), + } + }; + // A name too long for the column counts as absent: one bad entry must not fail the + // upsert and end the rest of the sync. + let display_name = match &rt.display_name { + Some(Some(name)) if name.chars().count() > 100 => { + tracing::warn!( + "Ignoring the display_name of resource type {}: longer than 100 characters", + rt.name + ); + stored.and_then(|s| s.display_name.clone()) + } + Some(name) => name.clone(), + None => stored.and_then(|s| s.display_name.clone()), + }; + ResourceTypeRow { + schema: rt.schema.clone(), + description: rt.description.clone(), + format_extension, + is_fileset, + display_name, + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum SyncSource { + Hub, + ImageCache, +} + +pub struct SyncOutcome { + pub source: SyncSource, + /// Every type the source listed, changed or not. + pub listed: Vec, + /// The types this sync created or updated. + pub changed: Vec, +} + +impl SyncOutcome { + pub fn summary(&self) -> String { + let synced = self.changed.len(); + let unchanged = self.listed.len() - synced; + match self.source { + SyncSource::Hub => { + format!("Synced {synced} resource types from the hub ({unchanged} unchanged)") + } + SyncSource::ImageCache => format!( + "Synced {synced} resource types from the image's cache ({unchanged} unchanged)" + ), + } + } +} + +async fn apply(db: &DB, source: SyncSource, types: &[HubResourceType]) -> Result { + let stored: HashMap = sqlx::query_as::< + _, + ( + String, + Option, + Option, + Option, + bool, + Option, + ), + >( + "SELECT name, schema, description, format_extension, is_fileset, display_name + FROM resource_type WHERE workspace_id = 'admins'", + ) + .fetch_all(db) + .await? + .into_iter() + .map( + |(name, schema, description, format_extension, is_fileset, display_name)| { + ( + name, + ResourceTypeRow { schema, description, format_extension, is_fileset, display_name }, + ) + }, + ) + .collect(); + + let mut changed = vec![]; + for rt in types { + let current = stored.get(&rt.name); + let target = target_row(rt, current); + if current == Some(&target) { + continue; + } + sqlx::query( + "INSERT INTO resource_type + (workspace_id, name, schema, description, format_extension, is_fileset, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $5, $6, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + format_extension = EXCLUDED.format_extension, is_fileset = EXCLUDED.is_fileset, + display_name = EXCLUDED.display_name, edited_at = now()", + ) + .bind(&rt.name) + .bind(&target.schema) + .bind(&target.description) + .bind(&target.format_extension) + .bind(target.is_fileset) + .bind(&target.display_name) + .execute(db) + .await?; + changed.push(rt.name.clone()); + } + + Ok(SyncOutcome { source, listed: types.iter().map(|rt| rt.name.clone()).collect(), changed }) +} + +#[derive(Deserialize, Default)] +struct SyncState { + last_attempt_at: Option>, + last_hub_sync_at: Option>, +} + +async fn load_state(db: &DB) -> Result { + let value: Option = + sqlx::query_scalar("SELECT value FROM background_task_state WHERE name = $1") + .bind(SYNC_STATE_TASK) + .fetch_optional(db) + .await?; + Ok(value + .and_then(|v| serde_json::from_value(v).ok()) + .unwrap_or_default()) +} + +/// Merged into the stored state, so a writer only ever sets the keys it names. +async fn record_state(db: &DB, patch: serde_json::Value) -> Result<()> { + sqlx::query( + "INSERT INTO background_task_state (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE + SET value = background_task_state.value || EXCLUDED.value, updated_at = now()", + ) + .bind(SYNC_STATE_TASK) + .bind(patch) + .execute(db) + .await?; + Ok(()) +} + +async fn setting_enabled(db: &DB, name: &str) -> Result { + Ok(load_value_from_global_settings(db, name) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false)) +} + +/// Syncs from the configured hub and records the attempt, successful or not. +async fn sync_from_hub(db: &DB) -> Result { + let attempted_at = Utc::now(); + let result = match fetch_configured_hub(db).await { + Ok(types) => apply(db, SyncSource::Hub, &types).await, + Err(e) => Err(e), + }; + let state = match &result { + Ok(_) => { + json!({ "last_attempt_at": attempted_at, "last_hub_sync_at": attempted_at, "last_error": null }) + } + Err(e) => json!({ "last_attempt_at": attempted_at, "last_error": e.to_string() }), + }; + record_state(db, state).await?; + result +} + +/// Syncs from the configured hub, falling back to the image's cache when the hub is disabled +/// or unreachable and the instance has never synced from a hub. +pub async fn sync(db: &DB) -> Result { + let hub_err = if setting_enabled(db, DISABLE_HUB_SETTING).await? { + Error::BadRequest("The hub is disabled on this instance".to_string()) + } else { + match sync_from_hub(db).await { + Ok(outcome) => return Ok(outcome), + Err(e) => e, + } + }; + match fall_back_to_image_cache(db, &cache_path()).await? { + Some(outcome) => { + tracing::warn!("Hub resource type sync failed, used the image's cache: {hub_err}"); + Ok(outcome) + } + None => Err(hub_err), + } +} + +/// The cache is the listing as of the image's build, so standing in for the hub on an +/// instance that has synced from one would revert every type the hub has updated since. +async fn fall_back_to_image_cache(db: &DB, path: &str) -> Result> { + if let Some(at) = load_state(db).await?.last_hub_sync_at { + tracing::info!( + "Not applying the image's cached resource types: synced from the hub at {at}" + ); + return Ok(None); + } + sync_from_cache_file(db, path).await +} + +/// Applies the listing baked into the image (`SYNC_CACHED_RT`), on every start and whether or +/// not the instance has synced from a hub: it is how an instance that cannot reach the public +/// hub, a private hub listing only its own types included, takes each new image's public types. +/// `None` when the image carries no cache. +pub async fn sync_from_image_cache(db: &DB) -> Result> { + sync_from_cache_file(db, &cache_path()).await +} + +async fn sync_from_cache_file(db: &DB, path: &str) -> Result> { + let content = match tokio::fs::read_to_string(path).await { + Ok(content) => content, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::info!("No cached resource types at {path}"); + return Ok(None); + } + Err(e) => { + return Err(Error::InternalErr(format!( + "Failed to read cached resource types at {path}: {e}" + ))) + } + }; + let types: Vec = serde_json::from_str(&content).map_err(|e| { + Error::InternalErr(format!( + "Failed to parse cached resource types at {path}: {e}" + )) + })?; + Ok(Some(apply(db, SyncSource::ImageCache, &types).await?)) +} + +/// While `sync_hub_resource_types_daily` is on, syncs once a day, from whichever server first +/// finds the sync due. Off by default: each sync overwrites local edits to the `admins` types +/// the hub also defines. The state it reads lives in the database, so servers starting or +/// restarting do not each sync again. +pub fn spawn_daily_sync(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { + tokio::spawn(async move { + // Leaves time for the hub URL and secret settings to load, and spreads out servers + // that start together. + let mut wait = std::time::Duration::from_secs(rand::random_range(30..90)); + loop { + tokio::select! { + _ = tokio::time::sleep(wait) => {} + _ = killpill_rx.recv() => return, + } + if let Err(e) = sync_if_due(&db).await { + tracing::error!("Hub resource type sync: {e:#}"); + } + wait = CHECK_INTERVAL; + } + }); +} + +async fn sync_if_due(db: &DB) -> Result<()> { + if !setting_enabled(db, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING).await? + || setting_enabled(db, DISABLE_HUB_SETTING).await? + { + return Ok(()); + } + // A transaction-scoped lock: a dropped transaction rolls back and releases it, so a + // pass cut short cannot leave it held on a pooled connection. The transaction only + // owns the lock; the sync runs on other connections. + let mut lock_tx = db.begin().await?; + let locked: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(SYNC_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await?; + if !locked { + return Ok(()); + } + let state = load_state(db).await?; + let now = Utc::now(); + let due = state + .last_hub_sync_at + .is_none_or(|at| now - at >= SYNC_INTERVAL) + && state + .last_attempt_at + .is_none_or(|at| now - at >= RETRY_INTERVAL); + // Hub only. Falling back to the image's cache here would re-apply the same listing every + // hour on an instance that never reaches a hub, over any edit made since; the cache is + // applied at boot (`SYNC_CACHED_RT`) or on demand instead. + if due { + match sync_from_hub(db).await { + Ok(outcome) if outcome.changed.is_empty() => tracing::info!("{}", outcome.summary()), + Ok(outcome) => { + tracing::info!("{}: {}", outcome.summary(), outcome.changed.join(", ")) + } + Err(e) => tracing::warn!("Hub resource type sync failed, retrying in an hour: {e}"), + } + } + lock_tx.rollback().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hub_type( + format_extension: Option>, + display_name: Option>, + is_fileset: Option, + ) -> HubResourceType { + HubResourceType { + name: "t".to_string(), + schema: None, + description: None, + format_extension: format_extension.map(|e| e.map(str::to_string)), + display_name: display_name.map(|n| n.map(str::to_string)), + is_fileset, + } + } + + fn stored( + format_extension: Option<&str>, + display_name: Option<&str>, + is_fileset: bool, + ) -> ResourceTypeRow { + ResourceTypeRow { + schema: None, + description: None, + format_extension: format_extension.map(str::to_string), + is_fileset, + display_name: display_name.map(str::to_string), + } + } + + #[test] + fn absent_keys_keep_the_stored_values_and_nulls_clear_them() { + let current = stored(Some("csv"), Some("T"), false); + let kept = target_row(&hub_type(None, None, None), Some(¤t)); + assert_eq!(kept, current); + let cleared = target_row(&hub_type(Some(None), Some(None), None), Some(¤t)); + assert_eq!(cleared, stored(None, None, false)); + } + + #[test] + fn a_fileset_never_takes_a_format_extension() { + let local_fileset = stored(None, None, true); + let row = target_row( + &hub_type(Some(Some("csv")), None, None), + Some(&local_fileset), + ); + assert_eq!(row, stored(None, None, true)); + let row = target_row(&hub_type(Some(Some("csv")), None, Some(true)), None); + assert_eq!(row, stored(None, None, true)); + } + + #[sqlx::test(migrations = "../migrations")] + async fn only_sync_cached_rt_applies_the_image_cache_after_a_hub_sync( + db: DB, + ) -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("resource_types.json"); + std::fs::write( + &path, + json!([{ "name": "cached_type", "schema": null, "description": "image copy" }]) + .to_string(), + )?; + let path = path.to_str().unwrap(); + let description = || { + sqlx::query_scalar::<_, Option>( + "SELECT description FROM resource_type WHERE workspace_id = 'admins' AND name = 'cached_type'", + ) + .fetch_one(&db) + }; + + assert!(fall_back_to_image_cache(&db, path).await?.is_some()); + assert_eq!(description().await?.as_deref(), Some("image copy")); + + record_state(&db, json!({ "last_hub_sync_at": Utc::now() })).await?; + sqlx::query( + "UPDATE resource_type SET description = 'hub copy' WHERE workspace_id = 'admins' AND name = 'cached_type'", + ) + .execute(&db) + .await?; + assert!(fall_back_to_image_cache(&db, path).await?.is_none()); + assert_eq!(description().await?.as_deref(), Some("hub copy")); + + assert!(sync_from_cache_file(&db, path).await?.is_some()); + assert_eq!(description().await?.as_deref(), Some("image copy")); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations")] + async fn no_daily_sync_unless_turned_on(db: DB) -> anyhow::Result<()> { + sync_if_due(&db).await?; + + let state: Option = + sqlx::query_scalar("SELECT value FROM background_task_state WHERE name = $1") + .bind(SYNC_STATE_TASK) + .fetch_optional(&db) + .await?; + assert_eq!(state, None); + Ok(()) + } +} diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index a4164cd2c5..7c76fab2ee 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -267,6 +267,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub disable_hub: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub sync_hub_resource_types_daily: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_on_deploy: Option, #[serde(skip_serializing_if = "Option::is_none")] pub ai_sessions_instance_storage_fallback: Option, @@ -281,6 +283,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub github_app_webhook_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub api_base_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub email_domain: Option, #[serde(skip_serializing_if = "Option::is_none")] pub hub_base_url: Option, @@ -963,6 +967,9 @@ pub struct WorkerGroupConfig { pub autoscaling: Option, #[serde(skip_serializing_if = "Option::is_none")] pub native_mode: Option, + /// Workers of the group pull no new job while this is set; queued jobs wait. + #[serde(skip_serializing_if = "Option::is_none")] + pub paused: Option, /// Object store this group's dependency cache uses instead of the instance one. Same shape /// as the instance `object_store_cache_config`. #[serde(skip_serializing_if = "Option::is_none")] @@ -1008,6 +1015,43 @@ pub struct ConfigsDiff { pub deletes: Vec, } +/// Told of each write an instance config apply makes, as soon as it has landed: the writes +/// are not transactional, so an apply that fails midway must not leave the ones it did make +/// unrecorded. The audit entry is the implementor's to write, as this crate sits below +/// `windmill-audit`. +pub trait ConfigWriteObserver: Sync { + /// `value` is `None` for a deleted setting. + fn setting_written( + &self, + key: &str, + value: Option<&serde_json::Value>, + ) -> impl std::future::Future> + Send; + + fn worker_config_written( + &self, + group: &str, + deleted: bool, + ) -> impl std::future::Future> + Send; +} + +impl ConfigWriteObserver for () { + async fn setting_written( + &self, + _key: &str, + _value: Option<&serde_json::Value>, + ) -> crate::error::Result<()> { + Ok(()) + } + + async fn worker_config_written( + &self, + _group: &str, + _deleted: bool, + ) -> crate::error::Result<()> { + Ok(()) + } +} + /// Settings that must never be deleted by the operator or bulk API. pub const PROTECTED_SETTINGS: &[&str] = &[ "ducklake_user_pg_pwd", @@ -1019,6 +1063,8 @@ pub const PROTECTED_SETTINGS: &[&str] = &[ "rsa_keys", "jwt_secret", "min_keep_alive_version", + // Deleting it would let a worker sign OIDC tokens with a local key again. + "oidc_external_signer", ]; /// Secrets the server signs with or generated for itself. `jwt_secret` signs API and job @@ -1072,6 +1118,8 @@ pub const HIDDEN_SETTINGS: &[&str] = &[ "custom_instance_replication_pwd", // Same for the passwords and database registry Windmill keeps for the external cluster. "external_instance_pg_state", + // Written by servers that sign OIDC tokens with an external signer. + "oidc_external_signer", ]; /// Top-level settings whose entire value is sensitive and must be fully redacted in logs. @@ -1372,8 +1420,8 @@ pub fn diff_worker_configs( } /// Declaratively replace the global settings, rejecting a -/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the -/// API would reject. +/// `github_app_webhook_base_url`, `api_base_url` or +/// `http_route_default_allowed_origins` the API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1391,34 +1439,44 @@ pub fn diff_worker_configs( /// AUTHORIZATION: replaces instance-wide settings and takes no authed context, so /// callers MUST have established superadmin or equivalent system authority (the CLI /// and the operator both run with direct instance credentials). +/// +/// `observer` is what audits the sync: no user request stands behind these writes, so +/// nothing else records them. pub async fn sync_global_settings_declarative( db: &sqlx::Pool, current: &BTreeMap, desired: &BTreeMap, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { - let webhook_key = crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING; - // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a - // bool/number/object through as if the key were absent, and the diff below would - // then persist it — where the HTTP path answers "must be a URL string". - match desired.get(webhook_key) { - None | Some(serde_json::Value::Null) => {} - Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} - Some(serde_json::Value::String(s)) => crate::global_settings::validate_webhook_base_url(s) - .map_err(|e| anyhow::anyhow!("{webhook_key}: {e}"))?, - // Names the JSON kind rather than printing it: this is the last message on - // this path that could report submitted content, and an object or array could - // carry a secret into `sync-config` output and operator logs. - Some(other) => { - let kind = match other { - serde_json::Value::Bool(_) => "a boolean", - serde_json::Value::Number(_) => "a number", - serde_json::Value::Array(_) => "an array", - serde_json::Value::Object(_) => "an object", - _ => "a non-string value", - }; - return Err(anyhow::anyhow!( - "{webhook_key} must be a URL string, got {kind}" - )); + for url_key in [ + crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING, + crate::global_settings::API_BASE_URL_SETTING, + ] { + // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a + // bool/number/object through as if the key were absent, and the diff below would + // then persist it — where the HTTP path answers "must be a URL string". + match desired.get(url_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(serde_json::Value::String(s)) => { + crate::global_settings::validate_webhook_base_url(s) + .map_err(|e| anyhow::anyhow!("{url_key}: {e}"))? + } + // Names the JSON kind rather than printing it: this is the last message on + // this path that could report submitted content, and an object or array could + // carry a secret into `sync-config` output and operator logs. + Some(other) => { + let kind = match other { + serde_json::Value::Bool(_) => "a boolean", + serde_json::Value::Number(_) => "a number", + serde_json::Value::Array(_) => "an array", + serde_json::Value::Object(_) => "an object", + _ => "a non-string value", + }; + return Err(anyhow::anyhow!( + "{url_key} must be a URL string, got {kind}" + )); + } } } @@ -1452,16 +1510,22 @@ pub async fn sync_global_settings_declarative( .map_err(|e| anyhow::anyhow!("{max_expiration_key}: {e}"))?; let mut diff = diff_global_settings(current, desired, ApplyMode::Replace); - crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff).await?; - apply_settings_diff(db, &diff).await?; + crate::external_instance_pg::write_external_instance_pg_from_diff(db, &mut diff, observer) + .await?; + apply_settings_diff(db, &diff, observer).await?; Ok(()) } -/// Apply a settings diff to the global_settings table. +/// Apply a settings diff to the global_settings table, reporting each write to `observer`. +/// +/// AUTHORIZATION: writes instance-wide settings, credentials included, and checks +/// nothing; `observer` records and authorizes nothing. Callers MUST have established +/// superadmin or equivalent system authority. pub async fn apply_settings_diff( db: &sqlx::Pool, diff: &SettingsDiff, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { for (key, value) in &diff.upserts { sqlx::query( @@ -1484,6 +1548,7 @@ pub async fn apply_settings_diff( format_setting_value(key, value) ); } + observer.setting_written(key, Some(value)).await?; } for key in &diff.deletes { sqlx::query("DELETE FROM global_settings WHERE name = $1") @@ -1498,6 +1563,7 @@ pub async fn apply_settings_diff( } else { tracing::info!("Deleted global setting: {key}"); } + observer.setting_written(key, None).await?; } if diff.unchanged_count > 0 { tracing::info!("{} global setting(s) unchanged", diff.unchanged_count); @@ -1505,10 +1571,16 @@ pub async fn apply_settings_diff( Ok(()) } -/// Apply a configs diff to the config table (worker__ prefix). +/// Apply a configs diff to the config table (worker__ prefix), reporting each write to +/// `observer`. +/// +/// AUTHORIZATION: writes worker group configs, whose init scripts run on the workers, and +/// checks nothing; `observer` records and authorizes nothing. Callers MUST have +/// established superadmin or equivalent system authority. pub async fn apply_configs_diff( db: &sqlx::Pool, diff: &ConfigsDiff, + observer: &impl ConfigWriteObserver, ) -> anyhow::Result<()> { for (group_name, config_value) in &diff.upserts { let db_key = format!("worker__{group_name}"); @@ -1521,6 +1593,7 @@ pub async fn apply_configs_diff( .execute(db) .await?; tracing::info!("Synced worker config: {db_key}"); + observer.worker_config_written(group_name, false).await?; } for group_name in &diff.deletes { let db_key = format!("worker__{group_name}"); @@ -1529,6 +1602,7 @@ pub async fn apply_configs_diff( .execute(db) .await?; tracing::info!("Deleted worker config: {db_key}"); + observer.worker_config_written(group_name, true).await?; } Ok(()) } @@ -1650,7 +1724,11 @@ impl InstanceConfig { /// /// Reads the current state, computes a diff, then applies upserts and /// deletes for both global settings and worker configs. - pub async fn sync_to_db(&self, db: &sqlx::Pool) -> anyhow::Result<()> { + pub async fn sync_to_db( + &self, + db: &sqlx::Pool, + observer: &impl ConfigWriteObserver, + ) -> anyhow::Result<()> { let desired_settings = self.global_settings.to_settings_map(); // Current global settings @@ -1660,7 +1738,8 @@ impl InstanceConfig { .await?; let current_settings: BTreeMap = rows.into_iter().collect(); - sync_global_settings_declarative(db, ¤t_settings, &desired_settings).await?; + sync_global_settings_declarative(db, ¤t_settings, &desired_settings, observer) + .await?; // Worker configs let desired_configs: BTreeMap = self @@ -1688,7 +1767,7 @@ impl InstanceConfig { let configs_diff = diff_worker_configs(¤t_configs, &desired_configs, ApplyMode::Replace); - apply_configs_diff(db, &configs_diff).await?; + apply_configs_diff(db, &configs_diff, observer).await?; Ok(()) } diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 1a314d34c0..75670fe874 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -474,6 +474,8 @@ pub struct RunInlinePreviewScriptFnParams { pub created_by: String, pub permissioned_as: String, pub permissioned_as_email: String, + /// The caller's recorded end user: `job_perms.end_user_email` of the calling job. + pub end_user_email: Option, pub base_internal_url: String, pub worker_name: String, pub conn: crate::worker::Connection, @@ -496,6 +498,8 @@ pub struct RunInlineScriptFnParams { pub created_by: String, pub permissioned_as: String, pub permissioned_as_email: String, + /// The caller's recorded end user: `job_perms.end_user_email` of the calling job. + pub end_user_email: Option, pub base_internal_url: String, pub worker_name: String, pub conn: crate::worker::Connection, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 5754b8c8c1..58a477d7b9 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -75,6 +75,7 @@ pub mod flow_status; pub mod flows; pub mod folders; pub mod global_settings; +pub mod hub_resource_types; pub mod guest_jwt; pub mod indexer; pub mod instance_config; @@ -101,6 +102,8 @@ pub mod oauth2; pub mod oidc_ee; #[cfg(all(feature = "enterprise", feature = "openidconnect"))] pub mod oidc_oss; +#[cfg(all(feature = "enterprise", feature = "openidconnect", feature = "private"))] +pub mod oidc_signer_ee; #[cfg(feature = "private")] pub mod otel_ee; pub mod otel_oss; @@ -123,6 +126,7 @@ pub mod query_builders; pub mod queue; pub mod queue_metrics; pub mod result_stream; +pub mod runnable_job_stats; pub mod runnable_settings; pub mod runnables; pub mod schedule; @@ -153,6 +157,7 @@ pub mod wac; pub mod webhook; pub mod worker; pub mod worker_group_job_stats; +pub mod workspace_crypt; pub mod workspaces; pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50; diff --git a/backend/windmill-common/src/log_context.rs b/backend/windmill-common/src/log_context.rs index cdb1e27aea..c5cd191da8 100644 --- a/backend/windmill-common/src/log_context.rs +++ b/backend/windmill-common/src/log_context.rs @@ -49,6 +49,7 @@ pub struct LogContext { // Worker / job span (windmill-worker/src/worker.rs) pub worker: Option, + pub worker_group: Option, pub hostname: Option, pub tag: Option, pub job_id: Option, diff --git a/backend/windmill-common/src/min_version.rs b/backend/windmill-common/src/min_version.rs index 930b718b7e..05b664b23d 100644 --- a/backend/windmill-common/src/min_version.rs +++ b/backend/windmill-common/src/min_version.rs @@ -23,6 +23,11 @@ pub const MIN_VERSION_SUPPORTS_BUN_LOCKFILE_V2: VC = vc(1, 794, 0, "Bun v2 lockf // them, so a restriction set while one is live is silently ignored on the jobs it runs. Must // name the release this ships in. pub const MIN_VERSION_SUPPORTS_JOB_TOKEN_SCOPES: VC = vc(1, 822, 0, "Restricted job tokens"); +// Gates writing AES-256-GCM workspace ciphertext and the background pass that rewrites the +// stored secret variables. Reading it ships one release earlier (1.831), so a rollback to +// that release still reads every secret; below it, secrets written after the switch stay +// unreadable until rolled forward. Only worker versions are seen here, not servers. +pub const MIN_VERSION_SUPPORTS_AEAD_SECRETS: VC = vc(1, 832, 0, "AES-GCM secret encryption"); pub const MIN_VERSION_SUPPORTS_NODE_DEBOUNCING: VC = vc(1, 658, 0, "Flow node debouncing"); pub const MIN_VERSION_SUPPORTS_TOKEN_HASH: VC = vc(1, 659, 0, "Token hash storage"); pub const MIN_VERSION_SUPPORTS_SYNC_JOBS_DEBOUNCING: VC = vc(1, 602, 0, "Sync jobs debouncing"); diff --git a/backend/windmill-common/src/otel_oss.rs b/backend/windmill-common/src/otel_oss.rs index bd0a5202df..9dac8a07db 100644 --- a/backend/windmill-common/src/otel_oss.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -54,6 +54,19 @@ pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Opt None } +#[cfg(not(feature = "private"))] +pub(crate) fn init_logs_bridge_for_service( + _service: &str, + _hostname: &str, + _env: &str, +) -> Option { + None +} + +/// Export the log records still batched, for a process about to exit. +#[cfg(not(feature = "private"))] +pub async fn flush_otel_logs() {} + #[cfg(all(feature = "otel", feature = "enterprise", not(feature = "private")))] pub(crate) fn init_otlp_tracer( _mode: &Mode, @@ -132,3 +145,36 @@ pub fn otel_set_health_status_phase(_phase: &str) {} #[cfg(not(feature = "private"))] pub fn otel_set_health_db_unresponsive(_unresponsive: bool) {} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_memory( + _worker: &str, + _worker_group: &str, + _usage: Option, + _windmill_usage: Option, + _limit: Option, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_add_worker_cpu( + _worker: &str, + _worker_group: &str, + _usage_secs: f64, + _periods: u64, + _throttled_periods: u64, + _throttled_secs: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_occupancy_rate( + _worker: &str, + _worker_group: &str, + _window: &'static str, + _rate: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_record_job_memory_peak(_tag: &str, _bytes: u64) {} diff --git a/backend/windmill-common/src/runnable_job_stats.rs b/backend/windmill-common/src/runnable_job_stats.rs new file mode 100644 index 0000000000..fa1b99e215 --- /dev/null +++ b/backend/windmill-common/src/runnable_job_stats.rs @@ -0,0 +1,288 @@ +//! Hourly resource rollup per runnable and worker group (`runnable_job_stats`). +//! +//! Workers accumulate completed jobs in memory and upsert periodically, so that the +//! "heaviest scripts" view never has to aggregate `v2_job_completed`. +//! +//! What a row leaves out: executors that run in the worker process (SQL, native +//! TypeScript, ...) have no CPU reading, and for memory report nothing or, for some SQL +//! ones, the size of their result, as the job's `memory_peak` does. Memory is otherwise +//! the peak of the job's main process only, and CPU counts that process and the children +//! it waited for, and nothing for a job that was killed. +//! +//! Nothing here checks authorization: these functions write and prune every workspace's +//! rows, and are for the worker's completion path and the server's monitor only. Reads +//! go through the devops-gated API handler. + +use std::{collections::HashMap, sync::Mutex}; + +use sqlx::{Pool, Postgres}; + +use crate::{jobs::JobKind, worker_group_job_stats::get_current_hour}; + +/// Every run without a stable path is counted under this one, per workspace and worker +/// group: previews and other ad-hoc runs carry caller-chosen paths that would otherwise +/// grow the table without bound. +pub const ADHOC_RUNNABLE_PATH: &str = ""; + +/// Also the furthest back the API reads. +pub const RETENTION_DAYS: i64 = 30; + +#[derive(Debug, Default, Clone, Copy, PartialEq)] +struct RunnableStats { + job_count: i32, + total_duration_ms: i64, + max_memory_peak: i32, + sum_memory_peak: i64, + memory_sample_count: i32, + total_cpu_ms: i64, +} + +impl RunnableStats { + /// `mem_peak` is in kB and non-positive when the job had no reading. + fn add_job(&mut self, duration_ms: i64, mem_peak: i32, cpu_time_ms: Option) { + self.job_count += 1; + self.total_duration_ms += duration_ms; + if mem_peak > 0 { + self.max_memory_peak = self.max_memory_peak.max(mem_peak); + self.sum_memory_peak += mem_peak as i64; + self.memory_sample_count += 1; + } + self.total_cpu_ms += cpu_time_ms.unwrap_or(0).max(0); + } + + fn merge(&mut self, other: &RunnableStats) { + self.job_count += other.job_count; + self.total_duration_ms += other.total_duration_ms; + self.max_memory_peak = self.max_memory_peak.max(other.max_memory_peak); + self.sum_memory_peak += other.sum_memory_peak; + self.memory_sample_count += other.memory_sample_count; + self.total_cpu_ms += other.total_cpu_ms; + } +} + +/// (hour, workspace_id, runnable_path, worker_group) +type StatsKey = (i64, String, String, String); + +lazy_static::lazy_static! { + static ref STATS: Mutex> = Mutex::new(HashMap::new()); +} + +/// The path a job is rolled up under: its own for the kinds whose path names something +/// deployed (a flow step's is `/`), the ad-hoc bucket otherwise. +pub fn stats_path(kind: JobKind, runnable_path: Option<&str>) -> &str { + match (kind, runnable_path) { + ( + JobKind::Script + | JobKind::Script_Hub + | JobKind::FlowScript + | JobKind::FlowNode + | JobKind::AIAgent, + Some(path), + ) if !path.is_empty() => path, + _ => ADHOC_RUNNABLE_PATH, + } +} + +pub fn accumulate_runnable_job_stats( + workspace_id: &str, + kind: JobKind, + runnable_path: Option<&str>, + worker_group: &str, + duration_ms: i64, + mem_peak: i32, + cpu_time_ms: Option, +) { + let key = ( + get_current_hour(), + workspace_id.to_string(), + stats_path(kind, runnable_path).to_string(), + worker_group.to_string(), + ); + STATS + .lock() + .unwrap() + .entry(key) + .or_default() + .add_job(duration_ms, mem_peak, cpu_time_ms); +} + +/// Counts the time spent on a round of a job that is not completing, without counting a +/// job: a suspended workflow-as-code round replays its finished steps for free when it +/// resumes, so the time they took would otherwise never reach the rollup. A row can +/// therefore hold time and no job until the workflow completes. +pub fn accumulate_runnable_round( + workspace_id: &str, + kind: JobKind, + runnable_path: Option<&str>, + worker_group: &str, + duration_ms: i64, + cpu_time_ms: Option, +) { + let key = ( + get_current_hour(), + workspace_id.to_string(), + stats_path(kind, runnable_path).to_string(), + worker_group.to_string(), + ); + let mut stats = STATS.lock().unwrap(); + let entry = stats.entry(key).or_default(); + entry.total_duration_ms += duration_ms; + entry.total_cpu_ms += cpu_time_ms.unwrap_or(0).max(0); +} + +pub async fn flush_runnable_job_stats(db: &Pool) -> Result<(), sqlx::Error> { + let drained: Vec<(StatsKey, RunnableStats)> = { + let mut stats = STATS.lock().unwrap(); + if stats.is_empty() { + return Ok(()); + } + stats.drain().collect() + }; + + let mut hours = Vec::with_capacity(drained.len()); + let mut workspace_ids = Vec::with_capacity(drained.len()); + let mut runnable_paths = Vec::with_capacity(drained.len()); + let mut worker_groups = Vec::with_capacity(drained.len()); + let mut job_counts = Vec::with_capacity(drained.len()); + let mut durations = Vec::with_capacity(drained.len()); + let mut max_peaks = Vec::with_capacity(drained.len()); + let mut sum_peaks = Vec::with_capacity(drained.len()); + let mut sample_counts = Vec::with_capacity(drained.len()); + let mut cpus = Vec::with_capacity(drained.len()); + for ((hour, workspace_id, runnable_path, worker_group), s) in &drained { + hours.push(*hour); + workspace_ids.push(workspace_id.clone()); + runnable_paths.push(runnable_path.clone()); + worker_groups.push(worker_group.clone()); + job_counts.push(s.job_count); + durations.push(s.total_duration_ms); + max_peaks.push(s.max_memory_peak); + sum_peaks.push(s.sum_memory_peak); + sample_counts.push(s.memory_sample_count); + cpus.push(s.total_cpu_ms); + } + + // The join drops rows of a workspace deleted since its jobs ran. The ORDER BY makes + // every worker lock the rows they share in one order: without it two concurrent + // flushes can deadlock, and the join is free to undo an ordering of the arrays. + let res = sqlx::query!( + r#" + INSERT INTO runnable_job_stats + (hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms, + max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms) + SELECT s.hour, s.workspace_id, s.runnable_path, s.worker_group, s.job_count, + s.total_duration_ms, s.max_memory_peak, s.sum_memory_peak, + s.memory_sample_count, s.total_cpu_ms + FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::int[], $6::bigint[], + $7::int[], $8::bigint[], $9::int[], $10::bigint[]) + AS s(hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms, + max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms) + JOIN workspace w ON w.id = s.workspace_id + ORDER BY s.hour, s.workspace_id, s.runnable_path, s.worker_group + ON CONFLICT (hour, workspace_id, runnable_path, worker_group) DO UPDATE SET + job_count = runnable_job_stats.job_count + EXCLUDED.job_count, + total_duration_ms = runnable_job_stats.total_duration_ms + EXCLUDED.total_duration_ms, + max_memory_peak = GREATEST(runnable_job_stats.max_memory_peak, EXCLUDED.max_memory_peak), + sum_memory_peak = runnable_job_stats.sum_memory_peak + EXCLUDED.sum_memory_peak, + memory_sample_count = runnable_job_stats.memory_sample_count + EXCLUDED.memory_sample_count, + total_cpu_ms = runnable_job_stats.total_cpu_ms + EXCLUDED.total_cpu_ms + "#, + &hours, + &workspace_ids, + &runnable_paths, + &worker_groups, + &job_counts, + &durations, + &max_peaks, + &sum_peaks, + &sample_counts, + &cpus, + ) + .execute(db) + .await; + + if let Err(e) = res { + let mut stats = STATS.lock().unwrap(); + for (key, s) in drained { + stats.entry(key).or_default().merge(&s); + } + return Err(e); + } + Ok(()) +} + +/// Also removes the rows of workspaces that no longer exist: a flush that overlaps a +/// workspace's deletion still sees the workspace and can write after the deletion's own +/// cleanup ran. Such a row is at most a couple of hours old, so only the last day is +/// searched, which keeps both statements on a range of the primary key. +pub async fn cleanup_old_runnable_job_stats( + db: &Pool, + retention_days: i64, +) -> Result { + let current_hour = get_current_hour(); + let expired = sqlx::query!( + "DELETE FROM runnable_job_stats WHERE hour < $1", + current_hour - retention_days * 24 * 3600 + ) + .execute(db) + .await?; + let orphaned = sqlx::query!( + "DELETE FROM runnable_job_stats s WHERE s.hour >= $1 \ + AND NOT EXISTS (SELECT 1 FROM workspace w WHERE w.id = s.workspace_id)", + current_hour - 24 * 3600 + ) + .execute(db) + .await?; + Ok(expired.rows_affected() + orphaned.rows_affected()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_deployed_runnables_keep_their_path() { + assert_eq!( + stats_path(JobKind::Script, Some("f/etl/load")), + "f/etl/load" + ); + assert_eq!( + stats_path(JobKind::FlowScript, Some("f/etl/flow/a")), + "f/etl/flow/a" + ); + // A preview carries the path being edited, which is not a stable identity. + assert_eq!( + stats_path(JobKind::Preview, Some("f/etl/load")), + ADHOC_RUNNABLE_PATH + ); + assert_eq!( + stats_path(JobKind::FlowPreview, Some("f/etl/flow")), + ADHOC_RUNNABLE_PATH + ); + assert_eq!( + stats_path(JobKind::Dependencies, Some("f/etl/load")), + ADHOC_RUNNABLE_PATH + ); + assert_eq!(stats_path(JobKind::Script, None), ADHOC_RUNNABLE_PATH); + } + + #[test] + fn jobs_without_a_memory_reading_do_not_dilute_it() { + let mut s = RunnableStats::default(); + s.add_job(1000, 2048, Some(700)); + s.add_job(500, -3, None); + s.add_job(250, 0, Some(-5)); + s.add_job(250, 1024, Some(50)); + assert_eq!( + s, + RunnableStats { + job_count: 4, + total_duration_ms: 2000, + max_memory_peak: 2048, + sum_memory_peak: 3072, + memory_sample_count: 2, + total_cpu_ms: 750, + } + ); + } +} diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 01e75ac489..855e077004 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -220,6 +220,28 @@ pub fn initialize_tracing( (_guard, meter_provider) } +/// Tracing for a subcommand that writes to the instance without running a server +/// (`sync-config`, the operator): stdout, plus the OTel logs bridge when OTel logs are +/// enabled, which is how the audit events of its writes reach a collector. It writes no +/// service log files, as nothing lists such a process to read them back. +/// +/// Only audit events are exported. These processes log each setting they write with its +/// value, masked by a list of known secret fields rather than by shape, and that is fit +/// for the operator's own stdout but not for a collector. +pub fn initialize_subcommand_tracing(service: &str, hostname: &str, environment: &str) { + let logs_bridge = crate::otel_oss::init_logs_bridge_for_service(service, hostname, environment); + tracing_subscriber::registry() + .with(logs_bridge.with_filter(Targets::new().with_target(AUDIT_TARGET, LevelFilter::INFO))) + .with( + tracing_subscriber::fmt::layer().with_filter( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ), + ) + .init(); +} + lazy_static::lazy_static! { pub static ref LOG_COUNTING_BY_MIN: Arc>> = Arc::new(RwLock::new(HashMap::new())); } diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index af6bdfe2b3..116a15303e 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -281,6 +281,9 @@ pub fn sanitize_db_error(e: sqlx::Error) -> Error { return Error::from(e); }; match db_err.code().as_deref() { + Some("23514") if db_err.constraint() == Some("variable_value_length") => { + Error::BadRequest("A field exceeds its maximum length".to_string()) + } // check_violation — `proper_id` / `proper_name` and friends Some("23514") => Error::BadRequest( "Invalid path or name: it does not match the required format".to_string(), diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 1feb4182ab..58766cf25b 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -12,11 +12,13 @@ use crate::scripts::ScriptHash; use crate::secret_backend::{get_secret_value, is_external_stored_value}; use crate::utils::WarnAfterExt; use crate::worker::Connection; +use crate::workspace_crypt::CipherFormat; +pub use crate::workspace_crypt::WorkspaceCrypt; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; -use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; +use std::sync::atomic::{AtomicBool, Ordering}; lazy_static::lazy_static! { pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); @@ -211,7 +213,7 @@ pub struct CreateVariable { pub ws_specific: Option, } -pub async fn build_crypt(db: &DB, w_id: &str) -> crate::error::Result { +pub async fn build_crypt(db: &DB, w_id: &str) -> crate::error::Result { // Check cache first (300-second staleness) let cached_key_o = WORKSPACE_CRYPT_CACHE.get(w_id).and_then(|(ts, key)| { if ts > chrono::Utc::now().timestamp() - 300 { @@ -235,7 +237,7 @@ pub async fn build_crypt(db: &DB, w_id: &str) -> crate::error::Result crate::error::Result { +) -> crate::error::Result { let key = get_workspace_key(w_id, db).await?; Ok(crypt_from_key_with_suffix(&key, key_suffix)) } /// The cipher `build_crypt_with_key_suffix` builds, from a key string in hand. -pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> MagicCrypt256 { +pub fn crypt_from_key_with_suffix(key: &str, key_suffix: &str) -> WorkspaceCrypt { let crypt_key = if let Some(ref salt) = SECRET_SALT.as_ref() { format!("{}{}{}", key, salt, key_suffix) } else { format!("{}{}", key, key_suffix) }; - magic_crypt::new_magic_crypt!(crypt_key, 256) + WorkspaceCrypt::new(&crypt_key) } pub async fn get_workspace_key(w_id: &str, db: &DB) -> crate::error::Result { @@ -371,25 +373,178 @@ pub async fn get_secret_value_as_admin( Ok(r) } -pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { - mc.encrypt_str_to_base64(value) +pub fn encrypt(mc: &WorkspaceCrypt, value: &str) -> String { + mc.encrypt_str(value) } -pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result { - mc.decrypt_base64_to_string(value).map_err(|e| match e { - MagicCryptError::DecryptError(_) => error::Error::internal_err( - "Could not decrypt value. The value may have been encrypted with a different key." - .to_string(), - ), - _ => error::Error::internal_err(e.to_string()), +const DECRYPT_FAILED: &str = + "Could not decrypt value. The value may have been encrypted with a different key."; + +/// Every failure returns the same message so responses carry no signal about why decryption +/// failed (authentication, padding, base64, UTF-8); the kind of failure is only logged. +pub fn decrypt(mc: &WorkspaceCrypt, value: String) -> error::Result { + mc.decrypt_str(&value).map_err(|e| { + tracing::warn!("decryption failed: {e}"); + error::Error::internal_err(DECRYPT_FAILED.to_string()) }) } +static LEGACY_SECRET_UPGRADE_DONE: AtomicBool = AtomicBool::new(false); +static LEGACY_SECRET_UPGRADE_RUNNING: AtomicBool = AtomicBool::new(false); +const LEGACY_SECRET_UPGRADE_BATCH: i64 = 200; + +/// Whether this process still has to run [`upgrade_legacy_secret_variables`]: once new values +/// are written in v2, until one pass has gone through every workspace. +pub fn legacy_secret_upgrade_pending() -> bool { + CipherFormat::current() == CipherFormat::V2 + && !LEGACY_SECRET_UPGRADE_DONE.load(Ordering::Relaxed) +} + +/// Re-encrypt the secret variables still in the legacy format in v2, workspace by workspace. +/// Idempotent, and emits no git-sync event: the plaintexts do not change. +pub async fn upgrade_legacy_secret_variables(db: &DB) { + struct Running; + impl Drop for Running { + fn drop(&mut self) { + LEGACY_SECRET_UPGRADE_RUNNING.store(false, Ordering::Release); + } + } + if !legacy_secret_upgrade_pending() + || LEGACY_SECRET_UPGRADE_RUNNING.swap(true, Ordering::AcqRel) + { + return; + } + // Cleared on drop, so a pass that panics does not stop every later one in this process. + let _running = Running; + upgrade_legacy_secret_variables_pass(db).await; +} + +async fn upgrade_legacy_secret_variables_pass(db: &DB) { + let workspaces: Vec = match sqlx::query_scalar("SELECT id FROM workspace ORDER BY id") + .fetch_all(db) + .await + { + Ok(w) => w, + Err(e) => { + tracing::error!("secret encryption upgrade: listing workspaces failed: {e:#}"); + return; + } + }; + tracing::info!( + "secret encryption upgrade: re-encrypting legacy secret variables of {} workspaces in AES-256-GCM", + workspaces.len() + ); + let (mut upgraded, mut skipped, mut failed_workspaces) = (0, 0, 0); + for w_id in workspaces { + match upgrade_workspace_secret_variables(db, &w_id).await { + Ok((u, s)) => { + if u > 0 || s > 0 { + tracing::info!( + "secret encryption upgrade: workspace {w_id}: {u} re-encrypted, {s} skipped" + ); + } + upgraded += u; + skipped += s; + } + Err(e) => { + failed_workspaces += 1; + tracing::error!("secret encryption upgrade: workspace {w_id} failed: {e:#}"); + } + } + } + tracing::info!( + "secret encryption upgrade: done, {upgraded} re-encrypted, {skipped} skipped, {failed_workspaces} workspaces failed" + ); + if failed_workspaces == 0 { + LEGACY_SECRET_UPGRADE_DONE.store(true, Ordering::Relaxed); + } +} + +/// The number of values re-encrypted and of values skipped as they do not decrypt. +async fn upgrade_workspace_secret_variables(db: &DB, w_id: &str) -> error::Result<(usize, usize)> { + let (mut upgraded, mut skipped) = (0, 0); + let mut after = String::new(); + loop { + let mut tx = db.begin().await?; + // Shared, so a rotation (which locks the key row for update and rewrites every secret) + // never interleaves with a batch, and the key read here is the one the values are under. + let key: Option = sqlx::query_scalar( + "SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud' FOR SHARE", + ) + .bind(w_id) + .fetch_optional(&mut *tx) + .await?; + let Some(key) = key else { + return Ok((upgraded, skipped)); + }; + let mc = crypt_from_key_with_suffix(&key, ""); + let rows: Vec<(String, String)> = sqlx::query_as( + "SELECT path, value FROM variable WHERE workspace_id = $1 AND is_secret AND path > $2 \ + ORDER BY path LIMIT $3", + ) + .bind(w_id) + .bind(&after) + .bind(LEGACY_SECRET_UPGRADE_BATCH) + .fetch_all(&mut *tx) + .await?; + let last_page = (rows.len() as i64) < LEGACY_SECRET_UPGRADE_BATCH; + for (path, value) in rows { + if value.is_empty() || is_external_stored_value(&value) || mc.is_v2(&value) { + after = path; + continue; + } + match mc.upgrade(&value) { + Some(new_value) => { + // In a savepoint, so a value the update rejects (v2 is up to 44 chars longer + // than CBC, which a column narrower than 15040 cannot hold) is skipped + // without losing the batch. + let mut sp = sqlx::Acquire::begin(&mut tx).await?; + // Only over the value it was computed from, so a write racing it wins. + match sqlx::query( + "UPDATE variable SET value = $1 \ + WHERE workspace_id = $2 AND path = $3 AND value = $4", + ) + .bind(new_value) + .bind(w_id) + .bind(&path) + .bind(&value) + .execute(&mut *sp) + .await + { + Ok(r) => { + sp.commit().await?; + upgraded += r.rows_affected() as usize; + } + Err(e) => { + sp.rollback().await?; + tracing::warn!( + "secret encryption upgrade: secret variable {path} of workspace {w_id} could not be stored re-encrypted, leaving it as is: {e}" + ); + skipped += 1; + } + } + } + None => { + tracing::warn!( + "secret encryption upgrade: secret variable {path} of workspace {w_id} does not decrypt, leaving it as is" + ); + skipped += 1; + } + } + after = path; + } + tx.commit().await?; + if last_page { + return Ok((upgraded, skipped)); + } + } +} + pub const WM_SCHEDULED_FOR: &str = "WM_SCHEDULED_FOR"; lazy_static::lazy_static! { pub static ref CUSTOM_ENVS_CACHE: Cache)> = Cache::new(100); - pub static ref WORKSPACE_CRYPT_CACHE: Cache = Cache::new(1000); + pub static ref WORKSPACE_CRYPT_CACHE: Cache = Cache::new(1000); } @@ -595,7 +750,7 @@ pub async fn get_reserved_variables( ContextualVariable { name: "WM_END_USER_EMAIL".to_string(), value: end_user_email.unwrap_or_else(|| "".to_string()), - description: "Email of the end user that executed the current script, propagated to flow steps. Only set when the run was triggered from an app (empty otherwise).".to_string(), + description: "Email of the end user that executed the current script, even when it runs on behalf of another identity, propagated to flow steps and child jobs. Set for app and API runs (for a webhook, the owner of the token it was called with), empty for runs fired by a schedule or by a trigger such as an HTTP route or Kafka.".to_string(), is_custom: false, }, ContextualVariable { @@ -746,7 +901,39 @@ pub async fn get_variable_or_self_as( #[cfg(test)] mod tests { - use super::{can_bind_as_prologue_const, escape_js_single_quoted, is_valid_js_identifier}; + use super::{ + can_bind_as_prologue_const, decrypt, escape_js_single_quoted, is_valid_js_identifier, + DECRYPT_FAILED, + }; + use crate::error::Error; + use crate::workspace_crypt::WorkspaceCrypt; + use base64::Engine; + use magic_crypt::MagicCryptTrait; + + #[test] + fn decrypt_failures_all_return_the_same_message() { + let mc = WorkspaceCrypt::new("workspace-key"); + let legacy = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = WorkspaceCrypt::new("another-key"); + let b64 = base64::engine::general_purpose::STANDARD; + let v2 = mc.upgrade(&legacy.encrypt_str_to_base64("secret")).unwrap(); + let mut tampered = b64.decode(v2).unwrap(); + *tampered.last_mut().unwrap() ^= 1; + let failures = [ + "not base64 at all!".to_string(), + legacy.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]), + other.encrypt_str_to_base64("secret"), + b64.encode(&tampered), + ]; + for value in failures { + match decrypt(&mc, value.clone()) { + Err(Error::InternalErrLoc { message, .. }) => { + assert_eq!(message, DECRYPT_FAILED, "for {value:?}") + } + other => panic!("expected the uniform decrypt error for {value:?}, got {other:?}"), + } + } + } #[test] fn is_valid_js_identifier_gates_prologue_injection() { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index e68b81e4de..58223c5d4a 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -477,6 +477,7 @@ lazy_static::lazy_static! { env_vars: Default::default(), native_mode: false, object_store_cache_config: Default::default(), + paused: false, }); pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); @@ -844,11 +845,14 @@ pub fn make_suspended_pull_query(tags: &[String]) -> String { } // pub async fn make_suspended pub async fn store_suspended_pull_query(wc: &WorkerConfig) { - if wc.worker_tags.len() == 0 { - tracing::error!("Empty tags in worker tags, skipping"); - return; - } - let query = make_suspended_pull_query(&wc.worker_tags); + // An empty tag list must clear the query: keeping the previous one would leave a reloaded + // worker pulling suspended jobs for tags it no longer listens to. + let query = if wc.worker_tags.is_empty() { + tracing::error!("Empty tags in worker tags, pulling no suspended jobs"); + String::new() + } else { + make_suspended_pull_query(&wc.worker_tags) + }; WORKER_SUSPENDED_PULL_QUERY.store(std::sync::Arc::new(query)); } @@ -922,6 +926,42 @@ pub async fn store_pull_query(wc: &WorkerConfig) { WORKER_PULL_QUERIES_FAIRNESS.store(std::sync::Arc::new(fairness_queries)); } +const WORKER_GROUP_PAUSED_CACHE_TTL: Duration = Duration::from_secs(10); + +lazy_static::lazy_static! { + static ref WORKER_GROUP_PAUSED_CACHE: quick_cache::sync::Cache = + quick_cache::sync::Cache::new(1000); +} + +/// Whether the config of `worker_group` is paused, as the server sees it. A worker that reads +/// its own config knows from `WORKER_CONFIG`; this is for the server deciding on behalf of +/// workers that do not, and is cached since they ask on every pull. +pub async fn is_worker_group_paused(db: &DB, worker_group: &str) -> bool { + if let Some((paused, at)) = WORKER_GROUP_PAUSED_CACHE.get(worker_group) { + if at.elapsed() < WORKER_GROUP_PAUSED_CACHE_TTL { + return paused; + } + } + let paused = match sqlx::query_scalar!( + "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1", + format!("worker__{worker_group}") + ) + .fetch_optional(db) + .await + { + Ok(paused) => paused.flatten().unwrap_or(false), + Err(e) => { + tracing::error!("Could not read whether worker group {worker_group} is paused: {e:#}"); + return false; + } + }; + WORKER_GROUP_PAUSED_CACHE.insert( + worker_group.to_string(), + (paused, std::time::Instant::now()), + ); + paused +} + lazy_static::lazy_static! { pub static ref WINDMILL_DIR: String = { let dir = std::env::var("WINDMILL_DIR") @@ -1161,6 +1201,8 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub py314: bool, + pub py315: bool, pub sandbox: bool, } @@ -1743,6 +1785,76 @@ pub fn get_cpu_period() -> Option { Some(100000) } +/// Bucket bounds in bytes, 16 MiB to 32 GiB, shared by the Prometheus and OTel job +/// peak-memory histograms so the two exports stay comparable. +pub const JOB_MEMORY_PEAK_BUCKETS: [f64; 12] = { + let mut buckets = [0.0; 12]; + let mut i = 0; + while i < 12 { + buckets[i] = (16u64 << (20 + i)) as f64; + i += 1; + } + buckets +}; + +/// Cumulative CPU accounting of the worker's cgroup. The throttling fields stay at 0 +/// when the cgroup has no CPU controller, which is also what an unthrottled one reports. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct CpuStat { + pub usage_usec: u64, + pub nr_periods: u64, + pub nr_throttled: u64, + pub throttled_usec: u64, +} + +#[cfg(not(windows))] +fn cpu_stat_field(stat: &str, key: &str) -> Option { + stat.lines().find_map(|line| { + let (k, v) = line.split_once(' ')?; + (k == key).then(|| v.trim().parse().ok()).flatten() + }) +} + +#[cfg(not(windows))] +fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpu_stat_field(stat, "usage_usec")?, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_usec").unwrap_or(0), + }) +} + +/// cgroup v1 splits the same figures over two controllers and counts time in nanoseconds. +#[cfg(not(windows))] +fn parse_cgroup_v1_cpu_stat(cpuacct_usage: &str, stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpuacct_usage.trim().parse::().ok()? / 1000, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_time").unwrap_or(0) / 1000, + }) +} + +#[cfg(not(windows))] +pub fn get_cpu_stat() -> Option { + const V1_USAGE: &str = "/sys/fs/cgroup/cpuacct/cpuacct.usage"; + if Path::new(V1_USAGE).exists() { + parse_cgroup_v1_cpu_stat( + &std::fs::read_to_string(V1_USAGE).ok()?, + &std::fs::read_to_string("/sys/fs/cgroup/cpu/cpu.stat").unwrap_or_default(), + ) + } else { + let cgroup_path = get_cgroupv2_path()?; + parse_cgroup_v2_cpu_stat(&std::fs::read_to_string(format!("{cgroup_path}/cpu.stat")).ok()?) + } +} + +#[cfg(windows)] +pub fn get_cpu_stat() -> Option { + None +} + /// CPUs the process is allowed to run on, ignoring any bandwidth quota — the count /// Go's `NumCPU` reports. `available_parallelism` cannot stand in for it: that folds /// the quota in, so a fraction of a CPU makes it report a single-core machine. @@ -1912,6 +2024,7 @@ pub enum PingType { MainLoop, Job, InitScript, + Draining, } #[derive(Serialize, Deserialize)] pub struct Ping { @@ -1934,6 +2047,8 @@ pub struct Ping { pub occupancy_rate_30m: Option, pub job_isolation: Option, pub native_mode: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub draining: Option, pub ping_type: PingType, } pub async fn update_ping_http( @@ -1986,6 +2101,7 @@ pub async fn update_ping_http( insert_ping.memory, insert_ping.job_isolation, insert_ping.native_mode.unwrap_or(false), + insert_ping.draining, db, ) .await?; @@ -2014,10 +2130,21 @@ pub async fn update_ping_http( ) .await? } + PingType::Draining => set_worker_draining_query(worker_name, db).await?, } Ok(()) } +pub async fn set_worker_draining_query(worker_name: &str, db: &DB) -> anyhow::Result<()> { + sqlx::query!( + "UPDATE worker_ping SET draining = true WHERE worker = $1", + worker_name + ) + .execute(db) + .await?; + Ok(()) +} + #[derive(Debug, Serialize, Deserialize)] pub struct JobCancelled { pub canceled_by: String, @@ -2116,6 +2243,9 @@ pub async fn fetch_raw_script_from_app_query( /// `started_at` and `jobs_executed` are the only two columns carried over, being the /// continuity itself — plus `ip` for as long as `ip` is `None`, which means the external IP /// lookup has not resolved yet and the predecessor's address is still the best guess. +/// +/// `draining` is `Some(false)` from a worker starting up and `None` from one re-registering +/// mid-life, which must leave alone a flag its own shutdown may have set in the meantime. pub async fn insert_ping_query( worker_instance: &str, worker_name: &str, @@ -2129,6 +2259,7 @@ pub async fn insert_ping_query( memory: Option, job_isolation: Option, native_mode: bool, + draining: Option, db: &DB, ) -> anyhow::Result { // A NULL `ip` means the external IP lookup is still in flight; a later ping fills it in, and @@ -2136,8 +2267,8 @@ pub async fn insert_ping_query( // lookup that has failed reports `external_ip::UNRETRIEVABLE_IP`, which does overwrite it. The // literal below must stay equal to `external_ip::UNKNOWN_IP`. let previous_jobs_executed = sqlx::query_scalar!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12) ON CONFLICT (worker) - DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, dedicated_workers, wm_version, vcpus, memory, job_isolation, native_mode, draining) VALUES ($1, $2, COALESCE($3, 'NO IP'), $4, $5, $6, $7, $8, $9, $10, $11, $12, COALESCE($13, false)) ON CONFLICT (worker) + DO UPDATE set ping_at = now(), worker_instance = EXCLUDED.worker_instance, ip = COALESCE($3, worker_ping.ip), custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group, dedicated_worker = EXCLUDED.dedicated_worker, dedicated_workers = EXCLUDED.dedicated_workers, wm_version = EXCLUDED.wm_version, vcpus = COALESCE(EXCLUDED.vcpus, worker_ping.vcpus), memory = COALESCE(EXCLUDED.memory, worker_ping.memory), job_isolation = EXCLUDED.job_isolation, native_mode = EXCLUDED.native_mode, current_job_id = NULL, current_job_workspace_id = NULL, draining = COALESCE($13, worker_ping.draining) RETURNING jobs_executed", worker_instance, worker_name, @@ -2151,6 +2282,7 @@ pub async fn insert_ping_query( memory, job_isolation.as_deref(), native_mode, + draining, ) .fetch_one(db) .await?; @@ -2470,6 +2602,12 @@ pub async fn load_worker_config( .collect(); priority_tags_sorted.push(PriorityTags { priority: 0, tags: Vec::from_iter(all_tags_set) }); // push the tags that were not listed as high priority with a priority = 0 priority_tags_sorted.sort_by_key(|elt| Reverse(elt.priority)); // sort by priority DESC + + // The buckets come out of hash maps: without a fixed order two loads of the same + // config compare unequal, and a reload reads that as a config change. + for bucket in priority_tags_sorted.iter_mut() { + bucket.tags.sort(); + } } else { // if no priority is used, push all tags with a priority to 0 priority_tags_sorted.push(PriorityTags { priority: 0, tags: worker_tags.clone() }); @@ -2539,6 +2677,7 @@ pub async fn load_worker_config( env_vars: resolved_env_vars, native_mode, object_store_cache_config: config.object_store_cache_config, + paused: config.paused.unwrap_or(false), }) } @@ -2629,6 +2768,7 @@ pub struct WorkerConfigOpt { pub env_vars_allowlist: Option>, pub native_mode: Option, pub object_store_cache_config: Option, + pub paused: Option, } impl Default for WorkerConfigOpt { @@ -2648,6 +2788,7 @@ impl Default for WorkerConfigOpt { env_vars_allowlist: Default::default(), native_mode: Default::default(), object_store_cache_config: Default::default(), + paused: Default::default(), } } } @@ -2670,12 +2811,15 @@ pub struct WorkerConfig { /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate /// that parses it, and comparing the raw value is what tells a reload the store changed. pub object_store_cache_config: Option, + /// The group pulls no job from the queue: its workers finish what they hold, keep pinging + /// and leave queued jobs for when it is resumed. + pub paused: bool, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, object_store_cache_config: {} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, paused: {:?}, object_store_cache_config: {} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, self.paused, // holds bucket credentials self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None")) } @@ -2750,6 +2894,8 @@ pub enum PyVAlias { #[default] Py312, Py313, + Py314, + Py315, } impl Into for PyVAlias { @@ -2767,13 +2913,20 @@ impl Into for PyVAlias { impl PyVAlias { pub fn all>() -> Vec { use PyVAlias::*; - vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()] + vec![ + Py310.into(), + Py311.into(), + Py312.into(), + Py313.into(), + Py314.into(), + Py315.into(), + ] } // Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH) fn major(&self) -> u32 { use PyVAlias::*; match self { - Py310 | Py311 | Py312 | Py313 => 3, + Py310 | Py311 | Py312 | Py313 | Py314 | Py315 => 3, // Py400 | Py401 => 4 } } @@ -2788,6 +2941,8 @@ impl PyVAlias { "311" => Some(Py311), "312" => Some(Py312), "313" => Some(Py313), + "314" => Some(Py314), + "315" => Some(Py315), _ => None, } } @@ -2847,6 +3002,33 @@ mod tests { use super::*; use std::collections::HashMap; + #[cfg(not(windows))] + #[test] + fn cgroup_cpu_stat_is_normalized_to_microseconds() { + let v2 = "usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n\ + nr_periods 40\nnr_throttled 7\nthrottled_usec 350000\nnr_bursts 0\n"; + let expected = CpuStat { + usage_usec: 2_500_000, + nr_periods: 40, + nr_throttled: 7, + throttled_usec: 350_000, + }; + assert_eq!(parse_cgroup_v2_cpu_stat(v2), Some(expected)); + + let v1_stat = "nr_periods 40\nnr_throttled 7\nthrottled_time 350000000\n"; + assert_eq!( + parse_cgroup_v1_cpu_stat("2500000000\n", v1_stat), + Some(expected) + ); + + // No CPU controller: usage is still reported, throttling reads as zero. + assert_eq!( + parse_cgroup_v2_cpu_stat("usage_usec 12\nuser_usec 8\nsystem_usec 4\n"), + Some(CpuStat { usage_usec: 12, ..Default::default() }) + ); + assert_eq!(parse_cgroup_v2_cpu_stat("nr_periods 40\n"), None); + } + #[test] fn datatable_role_is_read_from_the_leading_comment_block() { let role = |code| SqlAnnotations::datatable_role(code); diff --git a/backend/windmill-common/src/workspace_crypt.rs b/backend/windmill-common/src/workspace_crypt.rs new file mode 100644 index 0000000000..a561060e32 --- /dev/null +++ b/backend/windmill-common/src/workspace_crypt.rs @@ -0,0 +1,255 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2026 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! The cipher behind `variables::encrypt`/`decrypt`. +//! +//! Values are written as AES-256-GCM ("v2"), standard base64 of +//! `MAGIC (4) || nonce (12) || AES-256-GCM(PKCS#7(plaintext)) || tag (16)`. +//! Values written before are AES-256-CBC (`magic_crypt`) and are still read. +//! +//! The plaintext is padded to the block size so a v2 value decodes to a multiple of 16 bytes, +//! like a CBC one: the CLI and `validate_already_encrypted_secret` tell ciphertext from +//! plaintext by exactly that, and an old CLI that took a v2 value for plaintext would push it +//! to be encrypted again, corrupting the secret. + +use aes_gcm::aead::{Aead, AeadCore, KeyInit, OsRng}; +use aes_gcm::{Aes256Gcm, Key, Nonce}; +use base64::Engine; +use hkdf::Hkdf; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; +use sha2::Sha256; + +use crate::min_version::MIN_VERSION_SUPPORTS_AEAD_SECRETS; + +pub const V2_MAGIC: [u8; 4] = *b"wmA2"; +const NONCE_LEN: usize = 12; +const TAG_LEN: usize = 16; +const BLOCK: usize = 16; + +const AEAD_KEY_INFO: &[u8] = b"windmill workspace secret aes-256-gcm v1"; + +const B64: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; + +/// Why a value did not decrypt. Only ever logged, never returned to a caller. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DecryptFailure { + Encoding, + Authentication, + Utf8, +} + +impl std::fmt::Display for DecryptFailure { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + DecryptFailure::Encoding => "not base64", + DecryptFailure::Authentication => "authentication or padding check failed", + DecryptFailure::Utf8 => "plaintext is not UTF-8", + }) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CipherFormat { + LegacyCbc, + V2, +} + +impl CipherFormat { + /// What new values are written as. Old workers read secrets straight from the database with + /// the CBC cipher alone, so v2 is written only once every live worker reads it. + pub fn current() -> Self { + if MIN_VERSION_SUPPORTS_AEAD_SECRETS.met_conservatively() { + CipherFormat::V2 + } else { + CipherFormat::LegacyCbc + } + } +} + +#[derive(Clone)] +pub struct WorkspaceCrypt { + legacy: MagicCrypt256, + aead: Aes256Gcm, +} + +/// Code that drives `magic_crypt` directly on this cipher (app link secrets, AI session +/// objects) keeps the legacy format it always had. +impl std::ops::Deref for WorkspaceCrypt { + type Target = MagicCrypt256; + fn deref(&self) -> &MagicCrypt256 { + &self.legacy + } +} + +impl WorkspaceCrypt { + /// `crypt_key` is the key material the CBC cipher was always built from (workspace key, salt, + /// suffix). The AEAD key is derived from it under its own label, so the two ciphers never + /// share a key. + pub fn new(crypt_key: &str) -> Self { + let mut aead_key = [0u8; 32]; + Hkdf::::new(None, crypt_key.as_bytes()) + .expand(AEAD_KEY_INFO, &mut aead_key) + .expect("32 bytes is a valid HKDF-SHA256 output length"); + Self { + legacy: magic_crypt::new_magic_crypt!(crypt_key, 256), + aead: Aes256Gcm::new(Key::::from_slice(&aead_key)), + } + } + + pub fn encrypt_str(&self, plaintext: &str) -> String { + self.encrypt_str_in(plaintext, CipherFormat::current()) + } + + pub fn encrypt_str_in(&self, plaintext: &str, format: CipherFormat) -> String { + match format { + CipherFormat::V2 => B64.encode(self.seal(plaintext.as_bytes())), + CipherFormat::LegacyCbc => self.legacy.encrypt_str_to_base64(plaintext), + } + } + + pub fn decrypt_str(&self, ciphertext: &str) -> Result { + let bytes = B64 + .decode(ciphertext) + .map_err(|_| DecryptFailure::Encoding)?; + String::from_utf8(self.decrypt_bytes(&bytes)?).map_err(|_| DecryptFailure::Utf8) + } + + /// Whether `ciphertext` is a v2 value under this key. + pub fn is_v2(&self, ciphertext: &str) -> bool { + B64.decode(ciphertext) + .is_ok_and(|bytes| self.open(&bytes).is_some()) + } + + /// The v2 form of a legacy `ciphertext`, checked to decrypt back to the same plaintext. + /// `None` for a v2 value and for anything that does not decrypt under this key. + pub fn upgrade(&self, ciphertext: &str) -> Option { + if self.is_v2(ciphertext) { + return None; + } + let plaintext = self.decrypt_str(ciphertext).ok()?; + let upgraded = self.encrypt_str_in(&plaintext, CipherFormat::V2); + (self.decrypt_str(&upgraded).ok()? == plaintext).then_some(upgraded) + } + + fn decrypt_bytes(&self, bytes: &[u8]) -> Result, DecryptFailure> { + match self.open(bytes) { + Some(plaintext) => Ok(plaintext), + // Also for a value with the magic: a CBC ciphertext starts with it one time in 2^32. + None => self + .legacy + .decrypt_bytes_to_bytes(bytes) + .map_err(|_| DecryptFailure::Authentication), + } + } + + fn seal(&self, plaintext: &[u8]) -> Vec { + let nonce = Aes256Gcm::generate_nonce(&mut OsRng); + let pad = BLOCK - plaintext.len() % BLOCK; + let mut padded = Vec::with_capacity(plaintext.len() + pad); + padded.extend_from_slice(plaintext); + padded.resize(plaintext.len() + pad, pad as u8); + let sealed = self + .aead + .encrypt(&nonce, padded.as_slice()) + .expect("AES-GCM encryption fails only past 64 GiB of plaintext"); + let mut out = Vec::with_capacity(V2_MAGIC.len() + NONCE_LEN + sealed.len()); + out.extend_from_slice(&V2_MAGIC); + out.extend_from_slice(&nonce); + out.extend_from_slice(&sealed); + out + } + + fn open(&self, data: &[u8]) -> Option> { + let body = data.strip_prefix(&V2_MAGIC)?; + if body.len() < NONCE_LEN + BLOCK + TAG_LEN || (body.len() - NONCE_LEN) % BLOCK != 0 { + return None; + } + let (nonce, sealed) = body.split_at(NONCE_LEN); + let mut padded = self.aead.decrypt(Nonce::from_slice(nonce), sealed).ok()?; + let pad = *padded.last()? as usize; + if pad == 0 + || pad > BLOCK + || padded[padded.len() - pad..] + .iter() + .any(|b| *b as usize != pad) + { + return None; + } + padded.truncate(padded.len() - pad); + Some(padded) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "workspace-key"; + + #[test] + fn v2_round_trips() { + let wc = WorkspaceCrypt::new(KEY); + for plain in [ + "", + "s", + "exactly sixteen!", + "a longer secret value with ünïcödé", + ] { + let a = wc.encrypt_str_in(plain, CipherFormat::V2); + assert_ne!(a, wc.encrypt_str_in(plain, CipherFormat::V2), "nonce reuse"); + assert!(wc.is_v2(&a)); + assert_eq!(wc.decrypt_str(&a).unwrap(), plain); + } + } + + #[test] + fn legacy_cbc_still_decrypts() { + let wc = WorkspaceCrypt::new(KEY); + let legacy = magic_crypt::new_magic_crypt!(KEY, 256).encrypt_str_to_base64("old secret"); + assert!(!wc.is_v2(&legacy)); + assert_eq!(wc.decrypt_str(&legacy).unwrap(), "old secret"); + let upgraded = wc.upgrade(&legacy).unwrap(); + assert!(wc.is_v2(&upgraded)); + assert_eq!(wc.decrypt_str(&upgraded).unwrap(), "old secret"); + assert_eq!(wc.upgrade(&upgraded), None); + } + + #[test] + fn magic_prefixed_cbc_still_decrypts() { + // Too rare to find by search, so built: with magic_crypt's zero IV and SHA-256(key) AES + // key, the first CBC block is AES(P1), so P1 = AES^-1(MAGIC || zeros). + let mut block = [0u8; 16]; + block[..4].copy_from_slice(&V2_MAGIC); + let mut plaintext = { + use aes_gcm::aes::cipher::{BlockDecrypt, KeyInit}; + use sha2::Digest; + let cipher = aes_gcm::aes::Aes256::new_from_slice(&Sha256::digest(KEY)).unwrap(); + let mut b = aes_gcm::aes::Block::clone_from_slice(&block); + cipher.decrypt_block(&mut b); + b.to_vec() + }; + plaintext.extend_from_slice(b"tail of the secret value"); + let ciphertext = magic_crypt::new_magic_crypt!(KEY, 256).encrypt_bytes_to_bytes(&plaintext); + assert!(ciphertext.starts_with(&V2_MAGIC)); + assert_eq!( + WorkspaceCrypt::new(KEY).decrypt_bytes(&ciphertext).unwrap(), + plaintext + ); + } + + #[test] + fn v2_passes_the_block_multiple_ciphertext_shape_check() { + let wc = WorkspaceCrypt::new(KEY); + for len in 0..64 { + let decoded = B64 + .decode(wc.encrypt_str_in(&"x".repeat(len), CipherFormat::V2)) + .unwrap(); + assert!(!decoded.is_empty() && decoded.len() % 16 == 0, "len {len}"); + } + } +} diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index b05bd0a704..61fec5f7aa 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -190,7 +190,7 @@ pub enum ObjectType { DatatableMigration, } -pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28994/sync-script-to-git-repo-windmill"; +pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/29151/sync-script-to-git-repo-windmill"; /// Hub script that applies a repository's state back into a workspace /// (the repo → Windmill / "pull" direction). Same script the UI runs from @@ -198,7 +198,7 @@ pub const LATEST_GIT_SYNC_SCRIPT_PATH: &str = "hub/28994/sync-script-to-git-repo /// ignores the slug, so the slug is kept free of characters that would be /// percent-encoded into the run URL (a `:` becomes `%3A`, which some hardened /// reverse proxies reject as double-encoding when the client re-encodes it). -pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/28993/git-sync-init-repository-windmill"; +pub const GIT_SYNC_PULL_SCRIPT_PATH: &str = "hub/29150/git-sync-init-repository-windmill"; /// Prefix used to identify fork workspaces. A workspace whose id starts with this string is a /// fork of another workspace. @@ -1073,7 +1073,7 @@ pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result .fetch_one(db) .await .map_err(|e| Error::internal_err(format!("counting billable seats of {w_id}: {e:#}")))?; - let (developers, operators) = if operator_can_build_flows(db, w_id).await? { + let (developers, operators) = if operator_builder_rights(db, w_id).await?.any() { (row.developers + row.operators, 0) } else { (row.developers, row.operators) @@ -1211,14 +1211,47 @@ lazy_static::lazy_static! { /// Every operator right of a workspace, read and cached together because they share one /// `operator_settings` column and one invalidation. The two groups have opposite polarity: -/// `builder_flows` is granted on request and costs a seat, the `manage` rights are held by +/// the builder rights are granted on request and cost a seat, the `manage` rights are held by /// default and cost nothing. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] pub struct OperatorRights { - pub builder_flows: bool, + pub builder: OperatorBuilderRights, pub manage: OperatorManageRights, } +/// The two things a workspace can let its operators compose. Granted independently: composing a +/// flow is fully checked server-side, while a full-code app is a browser-built bundle no check can +/// read, so an admin may reasonably want the first without the second. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct OperatorBuilderRights { + pub flows: bool, + pub apps: bool, +} + +/// Which capability a gate needs. Naming a kind at each call site is what keeps the two rights +/// from collapsing back into one: a gate that takes "either" authorizes an app write on a +/// flows-only workspace. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum BuilderKind { + Flows, + Apps, +} + +impl OperatorBuilderRights { + /// Whether either right is granted. Only for surfaces that are genuinely not per-kind: the + /// seat a membership costs, and listing drafts. + pub fn any(&self) -> bool { + self.flows || self.apps + } + + pub fn has(&self, kind: BuilderKind) -> bool { + match kind { + BuilderKind::Flows => self.flows, + BuilderKind::Apps => self.apps, + } + } +} + /// Writes an operator may perform unless the workspace withdraws them. Unlike the visibility /// flags beside them in `operator_settings`, these are enforced; unlike a right a workspace /// grants, they are held by default and cost no seat. @@ -1297,6 +1330,7 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result // defaults have to differ here rather than at the call sites. let row = sqlx::query!( "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\", + COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\", COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\", COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\" FROM workspace_settings WHERE workspace_id = $1", @@ -1312,7 +1346,7 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result let rights = row .map(|r| OperatorRights { - builder_flows: r.flows, + builder: OperatorBuilderRights { flows: r.flows, apps: r.apps }, manage: OperatorManageRights { schedules: r.schedules, triggers: r.triggers }, }) .unwrap_or_default(); @@ -1322,21 +1356,22 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result Ok(rights) } -/// Whether operators of this workspace may compose flows out of already-deployed runnables. Per -/// workspace, not per user: every operator gets it, and consumes a full seat for it. -pub async fn operator_can_build_flows(db: &DB, workspace_id: &str) -> Result { - Ok(operator_rights(db, workspace_id).await?.builder_flows) +/// What operators of this workspace may compose out of already-deployed runnables. Per workspace, +/// not per user: every operator gets whatever is granted, and consumes a full seat for it. +pub async fn operator_builder_rights(db: &DB, workspace_id: &str) -> Result { + Ok(operator_rights(db, workspace_id).await?.builder) } -/// Gate for a write only a workspace that granted builder rights lets operators perform. `action` -/// completes "Operators cannot {action} for security reasons". -pub async fn check_operator_can_build_flows( +/// Gate for a write only a workspace that granted the builder right covering `kind` lets operators +/// perform. `action` completes "Operators cannot {action} for security reasons". +pub async fn check_operator_can_build( db: &DB, workspace_id: &str, is_operator: bool, + kind: BuilderKind, action: &str, ) -> Result<()> { - if is_operator && !operator_can_build_flows(db, workspace_id).await? { + if is_operator && !operator_builder_rights(db, workspace_id).await?.has(kind) { return Err(Error::PermissionDenied(format!( "Operators cannot {action} for security reasons" ))); @@ -1344,14 +1379,15 @@ pub async fn check_operator_can_build_flows( Ok(()) } -/// Whether a membership consumes an operator (half) seat rather than an author seat. A builder -/// right makes an operator an author of deployable artifacts, so it weighs a full seat. +/// Whether a membership consumes an operator (half) seat rather than an author seat. Either +/// builder right makes an operator an author of deployable artifacts, so both weigh the same full +/// seat: there is no half-author. pub async fn consumes_operator_seat( db: &DB, workspace_id: &str, is_operator: bool, ) -> Result { - Ok(is_operator && !operator_can_build_flows(db, workspace_id).await?) + Ok(is_operator && !operator_builder_rights(db, workspace_id).await?.any()) } /// Invalidate the operator rights cache for a workspace @@ -1559,7 +1595,8 @@ pub struct DataTableReference { /// The access decision for one data table: which role a caller gets, and who may ask for each. #[derive(Deserialize, Serialize, Debug, Clone, Default)] pub struct DataTablePermissions { - /// A role id from the instance catalog, or `admin`. Absent = `admin`. + /// A role id from the instance catalog, `admin`, or + /// [`crate::datatable_roles::NO_DEFAULT_DATATABLE_ROLE`]. Absent = `admin`. #[serde(default, skip_serializing_if = "Option::is_none")] pub default_role: Option, /// Keyed by instance role id, plus the reserved `admin` key. A role absent from this map @@ -2156,9 +2193,13 @@ pub enum DatatableAccess<'a> { PermissionedAs { permissioned_as: &'a str, email: &'a str, + /// The job's viewer: MUST be what the server recorded in `job_perms.end_user_email`, + /// never an argument or anything else the job or its caller can set. + end_user_email: Option<&'a str>, }, - /// A job identified by id; its owner is read from `v2_job`. For agent workers and anything - /// else that authenticates as infrastructure rather than as the job's user. + /// A job identified by id; its owner is read from `v2_job` and its viewer from `job_perms`. + /// For agent workers and anything else that authenticates as infrastructure rather than as + /// the job's user. Job(uuid::Uuid), /// No identity established. Unpermissioned data tables resolve as before; permissioned ones /// are refused, so a caller predating this feature fails closed. @@ -2206,6 +2247,13 @@ pub async fn can_use_datatable_role_in_governing_workspace( /// /// The resolved role **logs in as itself**. Never `SET ROLE`: a script could `RESET ROLE` its way /// back to admin. +/// +/// A default role other than `admin` is a floor rather than the whole connection: a resolution +/// naming no role holds it when the job's own identity may use it, plus the roles whose tenant +/// lists name the job's viewer, as one login that is a member of each. With no role at all it is +/// refused. Being an admin adds none of the viewer's and `admin` is never among them, so a viewer +/// cannot be handed the connection that owns the tables and bypasses row-level security. A named +/// role is that role alone. pub async fn get_datatable_resource_from_db( db: &DB, w_id: &str, diff --git a/backend/windmill-common/tests/billing_workspace.rs b/backend/windmill-common/tests/billing_workspace.rs index 0eddeabf53..ccc013d47a 100644 --- a/backend/windmill-common/tests/billing_workspace.rs +++ b/backend/windmill-common/tests/billing_workspace.rs @@ -143,6 +143,20 @@ async fn paid_seats_and_fork_count(db: Pool) { (4, 0, 1, 5), "builder rights bill every operator as a developer, service accounts stay half a seat" ); + sqlx::query( + "UPDATE workspace_settings SET operator_settings = '{\"builder_apps\": true}' + WHERE workspace_id = 'seat-root'", + ) + .execute(&db) + .await + .unwrap(); + windmill_common::workspaces::invalidate_operator_rights_cache("seat-root"); + let breakdown = billable_seats(&db, "seat-root").await.unwrap(); + assert_eq!( + (breakdown.developers, breakdown.operators, breakdown.seats), + (4, 0, 5), + "the app right alone also bills every operator as a developer" + ); insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; diff --git a/backend/windmill-mcp/src/client/mod.rs b/backend/windmill-mcp/src/client/mod.rs index 863d6b0355..5d13a40314 100644 --- a/backend/windmill-mcp/src/client/mod.rs +++ b/backend/windmill-mcp/src/client/mod.rs @@ -70,7 +70,8 @@ impl McpClient { headers.insert(name, value); } _ => { - tracing::warn!("Invalid header: {}={}", key, value); + // The value may be a resolved secret: keep it out of the logs. + tracing::warn!("Invalid header: {}", key); } } } diff --git a/backend/windmill-operator/Cargo.toml b/backend/windmill-operator/Cargo.toml index 30a7c0bb7a..f847e4ab25 100644 --- a/backend/windmill-operator/Cargo.toml +++ b/backend/windmill-operator/Cargo.toml @@ -19,6 +19,7 @@ serde_json.workspace = true sqlx.workspace = true tracing.workspace = true windmill-common = { workspace = true, default-features = false } +windmill-audit.workspace = true anyhow.workspace = true thiserror.workspace = true kube.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 207c8ee176..68749d2b90 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -182,6 +182,15 @@ pub struct CanceledBy { pub reason: Option, } +/// What only the worker that ran a job knows about it, for the per-runnable rollup: an +/// agent worker's completion is processed by the server, whose own worker group is not +/// the one the job ran in. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobResourceUsage { + pub worker_group: String, + pub cpu_time_ms: Option, +} + #[derive(Debug, Serialize, Deserialize)] pub struct JobCompleted { pub job: MiniCompletedJob, @@ -196,6 +205,8 @@ pub struct JobCompleted { pub duration: Option, pub has_stream: Option, pub from_cache: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_usage: Option, #[serde(skip)] pub flow_runners: Option>, #[serde(skip)] @@ -359,6 +370,22 @@ ORDER BY depth, id .filter_map(|r| r.id.clone()) .collect_vec(); + // A native retry is the child of its first attempt, which has completed by the time the + // retry is queued, so the walk above stops before reaching it. + let tree = [&[job.id][..], &jobs_to_cancel[..]].concat(); + let pending_retries = sqlx::query_scalar!( + "SELECT q.id FROM v2_job_queue q + JOIN native_retry_attempt n ON n.job_id = q.id + JOIN v2_job r ON r.id = q.id + JOIN v2_job a ON a.id = r.parent_job + WHERE q.workspace_id = $2 AND a.parent_job = ANY($1)", + tree.as_slice(), + w_id + ) + .fetch_all(&mut *tx) + .await?; + jobs_to_cancel.extend(pending_retries); + jobs_to_cancel.reverse(); if !jobs_to_cancel.is_empty() { tracing::info!("Found {} child jobs to cancel", jobs_to_cancel.len()); @@ -1205,7 +1232,7 @@ pub async fn add_completed_job_pre_shaped_failure, -) -> Result<(), Error> { +) -> Result { record_failure_metrics(completed_job, worker_name).await; tracing::error!( @@ -1213,7 +1240,7 @@ pub async fn add_completed_job_pre_shaped_failure, ) -> Result { + add_completed_job_error_with_span( + db, + completed_job, + mem_peak, + canceled_by, + e, + worker_name, + flow_is_done, + duration, + ) + .await + .map(|(wrapped, _)| wrapped) +} + +/// `add_completed_job_error`, also handing back what the completed row holds. +pub async fn add_completed_job_error_with_span( + db: &Pool, + completed_job: &MiniCompletedJob, + mem_peak: i32, + canceled_by: Option, + e: serde_json::Value, + worker_name: &str, + flow_is_done: bool, + duration: Option, +) -> Result<(WrappedError, CompletedSpan), Error> { record_failure_metrics(completed_job, worker_name).await; let result = WrappedError { error: e }; @@ -1250,7 +1302,7 @@ pub async fn add_completed_job_error( completed_job.workspace_id, serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); - let _ = add_completed_job( + let (_, span) = add_completed_job( db, &completed_job, false, @@ -1265,7 +1317,7 @@ pub async fn add_completed_job_error( ) .warn_after_seconds(10) .await?; - Ok(result) + Ok((result, span)) } lazy_static::lazy_static! { @@ -1304,7 +1356,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64), Error> { +) -> Result<(Uuid, CompletedSpan), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -1338,7 +1390,7 @@ pub async fn add_completed_job( }; let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| { + let (opt_uuid, span, _skip_downstream_error_handlers, wac_parent_ready) = (|| { commit_completed_job( db, completed_job, @@ -1379,7 +1431,7 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration)); + return Ok((job_id, span)); } // Auto-resolve a retry chain that ultimately worked, from whichever of the two @@ -1391,7 +1443,18 @@ pub async fn add_completed_job( .then(|| completed_job.parent_job) .flatten() } else if !success && !skipped && retry_pending { - Some(completed_job.parent_job.unwrap_or(completed_job.id)) + let is_retry_attempt = sqlx::query_scalar!( + "SELECT attempt FROM native_retry_attempt WHERE job_id = $1", + completed_job.id, + ) + .fetch_optional(db) + .await + .unwrap_or_else(|e| { + tracing::error!("reading the retry marker of {}: {e:#}", completed_job.id); + None + }) + .is_some(); + Some(native_retry_root(completed_job, is_retry_attempt)) } else { None }; @@ -1409,7 +1472,7 @@ pub async fn add_completed_job( } #[cfg(feature = "cloud")] - apply_completed_job_cloud_usage(db, completed_job, duration); + apply_completed_job_cloud_usage(db, completed_job, span.last_run_ms); #[cfg(all(feature = "enterprise", feature = "private"))] crate::jobs_ee::apply_completed_job_error_handlers( @@ -1426,7 +1489,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration)) + Ok((completed_job.id, span)) } async fn commit_completed_job( @@ -1444,7 +1507,7 @@ async fn commit_completed_job( // True when a native script retry was enqueued for this failed attempt, i.e. // this is not the terminal attempt — schedule completion handlers must wait. retry_pending: bool, -) -> windmill_common::error::Result<(Option, i64, bool, bool)> { +) -> windmill_common::error::Result<(Option, CompletedSpan, bool, bool)> { // let start = std::time::Instant::now(); let job_id = completed_job.id; @@ -1511,11 +1574,11 @@ async fn commit_completed_job( && monitor_parent.is_none() && !is_scheduled { - let Some(duration) = completion.execute(&mut *db.acquire().await?).await? else { + let Some(completed) = completion.execute(&mut *db.acquire().await?).await? else { return Err(not_in_queue_error(db, job_id).await); }; - log_completed_job(completed_job, duration, success); - return Ok((None, duration, false, false)); + log_completed_job(completed_job, completed.duration_ms, success); + return Ok((None, completed, false, false)); } let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1523,7 +1586,8 @@ async fn commit_completed_job( // The parent's rows are locked ahead of the child's own queue row (see // `record_child_completion` for the order this must keep), so the duration it stamps is read // before the completion: the one the completed row will hold. `now()` is fixed for the - // transaction, and a completed row already there keeps its own duration. + // transaction, and a completed row already there keeps its own duration. For a child that + // itself parked it is the last run's, matching the last start its timeline entry carries. let mut wac_parent_ready = false; if let Some(parent_job) = wac_parent { let Some(duration) = sqlx::query_scalar!( @@ -1550,7 +1614,7 @@ async fn commit_completed_job( .await?; } - let Some(duration) = completion.execute(&mut *tx).await? else { + let Some(completed) = completion.execute(&mut *tx).await? else { return Err(not_in_queue_error(&mut *tx, job_id).await); }; @@ -1760,16 +1824,35 @@ async fn commit_completed_job( tx.commit().warn_after_seconds(10).await?; - log_completed_job(completed_job, duration, success); + log_completed_job(completed_job, completed.duration_ms, success); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); Ok(( None, - duration, + completed, _skip_downstream_error_handlers, wac_parent_ready, )) } +/// What a job's completion wrote. `started_at` and `duration_ms` are the completed row's. +/// `last_run_ms` is the time the job's last run held a worker, which is what gets metered; +/// it differs from `duration_ms` only for a job that parked. +#[derive(Debug, Clone, Copy)] +pub struct CompletedSpan { + pub started_at: Option>, + pub duration_ms: i64, + pub last_run_ms: i64, +} + +impl CompletedSpan { + /// Whether the row spans more than the run that started at `last_started_at`, i.e. the + /// job parked and resumed. A flow records such a step with the row's span, so the step + /// reads like the job does. + pub fn outlasts_run(&self, last_started_at: Option>) -> bool { + matches!((self.started_at, last_started_at), (Some(first), Some(last)) if first < last) + } +} + /// What a job's completion writes, for `Completion::execute`. struct Completion<'a> { completed_job: &'a MiniCompletedJob, @@ -1786,6 +1869,11 @@ impl Completion<'_> { /// Moves the job from the queue to the completed jobs and refreshes a flow step's parent /// ping, as one statement. Returns `None` when the job was no longer in the queue. /// + /// For a job that parked (a Workflow-as-Code parent, see `suspend_wac_parent`) the row's + /// `started_at` and `duration_ms` span the whole workflow from its first start, and the + /// last run goes to `extras.wac_last_segment_ms` for readers that sum worker time from + /// completed rows. + /// /// The completion takes its cancellation from the queue row it deletes, not only from /// `canceled_by`: that is what the worker last read, and the delete waits for a cancel still /// being written, so the deleted row is the final word on whether the job was canceled. @@ -1793,7 +1881,7 @@ impl Completion<'_> { /// It locks the queue row before the completed row's key. Any other writer completing a job /// (the monitor's zombie fallback, debounce) must take them in the same order, or the two /// deadlock. - async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { + async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { let Completion { completed_job, success, @@ -1824,16 +1912,20 @@ impl Completion<'_> { .then_some(completed_job.parent_job) .flatten() else { - return sqlx::query_scalar!( + return sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1844,8 +1936,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1861,9 +1956,11 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -1883,16 +1980,20 @@ impl Completion<'_> { // A canceled flow is pinged too: it is completed by its next transition like any other // flow, and the zombie flow monitor needs the ping to finish the cancel if that // transition is lost. - sqlx::query_scalar!( + sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1903,8 +2004,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1920,14 +2024,16 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ), parent_ping AS ( UPDATE v2_job_runtime r SET ping = now() FROM v2_job_queue q WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12 AND EXISTS (SELECT 1 FROM completed) ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -1994,7 +2100,7 @@ async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool, bool), Error>> { +) -> Option, CompletedSpan, bool, bool), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { @@ -2194,18 +2300,17 @@ async fn restart_job_if_perpetual_inner( /// resolution too; `ON CONFLICT DO NOTHING` keeps a human's note intact. /// /// Membership of the chain must be *proven* per row, never inferred from `parent_job` -/// alone: `root` is `job.parent_job.unwrap_or(job.id)`, so for a job launched with an -/// explicit `parent_job` (WAC inline children, SDK-launched children) `root` is the -/// *calling* job, and its other failed children are unrelated. Resolving those would hide -/// exactly the failures this feature exists to surface, so each row must be either +/// alone: the success-side caller passes the succeeding job's `parent_job`, which for a +/// job launched with an explicit parent (WAC inline children, SDK-launched children) is +/// the *calling* job, and its other failed children are unrelated. Resolving those would +/// hide exactly the failures this feature exists to surface, so each row must be either /// - a job carrying a `native_retry_attempt` marker under `root` (provably an attempt), or -/// - `root` itself as the original attempt, which is only provable when `root` is -/// parentless and unmarked. +/// - `root` itself as the original attempt: unmarked, and proven a chain root by the +/// marked success under it, since only a chain root is ever a marked job's parent +/// (see [`native_retry_root`]). /// /// Both arms also require the same `runnable_id` as the succeeding attempt, since every -/// attempt in a chain runs the same runnable. The deliberate cost is a miss, not an -/// over-reach: when the original attempt had a parent it is an unmarked sibling -/// indistinguishable from any other child of the caller, so it stays red. +/// attempt in a chain runs the same runnable. /// /// Nothing is trusted of the caller (this writes through a privileged pool): the gate is /// "some marked attempt under `root` running `runnable_id` has succeeded", evaluated in @@ -2249,7 +2354,7 @@ pub async fn resolve_retry_chain_if_succeeded( AND ( (j.parent_job = $1 AND EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id)) - OR (c.id = $1 AND j.parent_job IS NULL + OR (c.id = $1 AND NOT EXISTS (SELECT 1 FROM native_retry_attempt WHERE job_id = c.id)) ) ON CONFLICT (job_id) DO NOTHING", @@ -2303,6 +2408,18 @@ async fn eval_retry_if( false } +/// The first attempt of `job`'s native retry chain, which every later attempt is pushed +/// with as `parent_job` and whose id seeds theirs. A first attempt may have a parent of its +/// own (a script dispatched from another job), so only a marked attempt's parent is the root: +/// rooting at that outer parent would merge the chains of its retried children. +fn native_retry_root(job: &MiniCompletedJob, is_retry_attempt: bool) -> Uuid { + if is_retry_attempt { + job.parent_job.unwrap_or(job.id) + } else { + job.id + } +} + /// Native script retry. When a failed `Script` or `Script_Hub` job carries a retry /// policy (via `runnable_settings_handle`) and has attempts left, enqueue a fresh /// attempt of the same script after the policy's backoff delay — instead of having @@ -2346,6 +2463,18 @@ pub async fn maybe_enqueue_native_script_retry( if !policy.has_attempts() { return Ok(false); } + // Cancelling a job that has a parent (every retry attempt does) only flags its queue row, + // and the worker that pulls it completes it without a `canceled_by`: read the flag here. + let soft_canceled = sqlx::query_scalar!( + "SELECT canceled_by IS NOT NULL AS \"canceled!\" FROM v2_job_queue WHERE id = $1", + job.id, + ) + .fetch_optional(db) + .await? + .unwrap_or(false); + if soft_canceled { + return Ok(false); + } // Attempt counter for this job: its `native_retry_attempt` marker, or 0 for // the first (un-marked) attempt. The marker is persistent (unlike the queue @@ -2358,7 +2487,7 @@ pub async fn maybe_enqueue_native_script_retry( .fetch_optional(db) .await? .unwrap_or(0) as u32; - let root = job.parent_job.unwrap_or(job.id); + let root = native_retry_root(job, prev_attempts > 0); // Scheduled chains keep the schedule trigger so the terminal attempt drives // the schedule completion handlers (on_failure/on_success); `parent_job` // keeps every retry out of the per-occurrence handler counting queries. @@ -4202,6 +4331,7 @@ impl PulledJobResult { ))), result_columns: None, mem_peak: 0, + resource_usage: None, cached_res_path: None, token: "".to_string(), canceled_by: None, @@ -4223,6 +4353,7 @@ impl PulledJobResult { }))), result_columns: None, mem_peak: 0, + resource_usage: None, cached_res_path: None, token: "".to_string(), canceled_by: None, diff --git a/backend/windmill-queue/tests/native_retry_test.rs b/backend/windmill-queue/tests/native_retry_test.rs index 9b0e05a74c..4cf278cf83 100644 --- a/backend/windmill-queue/tests/native_retry_test.rs +++ b/backend/windmill-queue/tests/native_retry_test.rs @@ -561,4 +561,123 @@ mod native_retry { .unwrap(); assert_eq!(status, "failure", "resolving must not change job status"); } + + // A script dispatched from another job keeps that job as its parent, so its chain is + // rooted at the first attempt itself: sibling dispatches must not share retry ids, and + // a parented first attempt is still resolved once its retry succeeds. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn parented_first_attempts_root_their_own_chains( + db: Pool, + ) -> anyhow::Result<()> { + let retry = Retry { + constant: ConstantDelay { attempts: 1, seconds: 0 }, + exponential: Default::default(), + retry_if: None, + }; + let handle = insert_rs( + RunnableSettings { + debouncing_settings: None, + concurrency_settings: None, + retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?, + }, + &db, + ) + .await?; + let dispatcher = Uuid::new_v4(); + let (a, b) = (Uuid::new_v4(), Uuid::new_v4()); + for first in [a, b] { + let attempt = mini(first, Some(dispatcher), handle); + assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?); + } + assert!( + retry_by_attempt(&db, a, 1).await.is_some(), + "a's retry hangs off a" + ); + assert!( + retry_by_attempt(&db, b, 1).await.is_some(), + "b's retry is not mistaken for a's" + ); + + let (c, c_retry, c_sibling) = (Uuid::new_v4(), Uuid::new_v4(), Uuid::new_v4()); + seed_job(&db, c, Some(dispatcher), false, "failure").await; + seed_job(&db, c_sibling, Some(dispatcher), false, "failure").await; + seed_job(&db, c_retry, Some(c), true, "success").await; + windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, c, WS, None).await?; + windmill_queue::jobs::resolve_retry_chain_if_succeeded(&db, dispatcher, WS, None).await?; + let resolved: Vec = sqlx::query_scalar("SELECT job_id FROM job_resolution") + .fetch_all(&db) + .await?; + assert_eq!( + resolved, + vec![c], + "only the parented first attempt resolves, not its sibling" + ); + Ok(()) + } + + // Cancelling the job that dispatched a script reaches that script's pending retry, which + // hangs off a first attempt that has already completed. + #[sqlx::test(migrations = "../migrations", fixtures("base", "schedule_push"))] + async fn cancelling_the_caller_cancels_a_pending_retry( + db: Pool, + ) -> anyhow::Result<()> { + let retry = Retry { + constant: ConstantDelay { attempts: 2, seconds: 60 }, + exponential: Default::default(), + retry_if: None, + }; + let handle = insert_rs( + RunnableSettings { + debouncing_settings: None, + concurrency_settings: None, + retry_settings: RetrySettings::from(&retry).insert_cached(&db).await?, + }, + &db, + ) + .await?; + let (caller, first) = (Uuid::new_v4(), Uuid::new_v4()); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, kind, runnable_path) VALUES ($1, $2, 'script', $3)", + ) + .bind(caller) + .bind(WS) + .bind(SCRIPT) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running) VALUES ($1, $2, now(), true)", + ) + .bind(caller) + .bind(WS) + .execute(&db) + .await?; + seed_job(&db, first, Some(caller), false, "failure").await; + let attempt = mini(first, Some(caller), handle); + assert!(maybe_enqueue_native_script_retry(&db, &attempt, &None, &no_result).await?); + let (retry_id, ..) = retry_by_attempt(&db, first, 1) + .await + .expect("retry is queued"); + + let tx = db.begin().await?; + let (tx, _) = + windmill_queue::jobs::cancel_job("test-user", None, caller, WS, tx, &db, false, false) + .await?; + tx.commit().await?; + + let canceled_by: Option = + sqlx::query_scalar("SELECT canceled_by FROM v2_job_queue WHERE id = $1") + .bind(retry_id) + .fetch_one(&db) + .await?; + assert_eq!(canceled_by.as_deref(), Some("test-user")); + + // The worker that pulls the canceled retry completes it without a `canceled_by`. + let (_, _, _, _, retry_handle) = retry_by_attempt(&db, first, 1).await.unwrap(); + let canceled_retry = mini(retry_id, Some(first), retry_handle); + assert!( + !maybe_enqueue_native_script_retry(&db, &canceled_retry, &None, &no_result).await?, + "a canceled retry does not enqueue the next attempt" + ); + Ok(()) + } } diff --git a/backend/windmill-runtime-nativets/Cargo.toml b/backend/windmill-runtime-nativets/Cargo.toml index d0b4e97f19..22da97ab1d 100644 --- a/backend/windmill-runtime-nativets/Cargo.toml +++ b/backend/windmill-runtime-nativets/Cargo.toml @@ -22,16 +22,15 @@ deno_webidl.workspace = true deno_web.workspace = true deno_crypto.workspace = true deno_net.workspace = true -deno_console.workspace = true -deno_url.workspace = true deno_core.workspace = true deno_ast.workspace = true deno_tls.workspace = true deno_permissions.workspace = true -deno_io.workspace = true -deno_fs.workspace = true deno_telemetry.workspace = true deno_error.workspace = true +sys_traits.workspace = true +# Not used directly: depending on it is what makes the workspace pin bind. +v8.workspace = true winapi.workspace = true itertools.workspace = true @@ -58,14 +57,8 @@ deno_webidl.workspace = true deno_web.workspace = true deno_crypto.workspace = true deno_net.workspace = true -deno_console.workspace = true -deno_url.workspace = true deno_core.workspace = true deno_ast.workspace = true -deno_tls.workspace = true -deno_permissions.workspace = true -deno_io.workspace = true -deno_fs.workspace = true deno_telemetry.workspace = true deno_error.workspace = true winapi.workspace = true diff --git a/backend/windmill-runtime-nativets/build.rs b/backend/windmill-runtime-nativets/build.rs index 29a6dbf81b..fab6cbb1ed 100644 --- a/backend/windmill-runtime-nativets/build.rs +++ b/backend/windmill-runtime-nativets/build.rs @@ -1,107 +1,11 @@ use deno_ast::{MediaType, ParseParams}; -use deno_core::{ModuleCodeString, ModuleName, SourceMapData}; +use deno_core::{Extension, ModuleCodeString, ModuleName, SourceMapData}; use deno_error::JsErrorBox; -use deno_fetch::FetchPermissions; -use deno_net::NetPermissions; -use deno_web::{BlobStore, TimersPermission}; -use std::borrow::Cow; +use std::collections::HashSet; use std::env; +use std::fmt::Write as _; use std::io::Write; use std::path::{Path, PathBuf}; -use std::sync::Arc; - -pub struct PermissionsContainer; - -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - _path: Cow<'a, Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_write<'a>( - &mut self, - _path: Cow<'a, Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_net_vsock( - &mut self, - _cid: u32, - _port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } -} - -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -impl NetPermissions for PermissionsContainer { - fn check_read( - &mut self, - _p: &str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_write( - &mut self, - _p: &str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_net>( - &mut self, - _host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - fn check_write_path<'a>( - &mut self, - _p: Cow<'a, Path>, - _api_name: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - fn check_vsock( - &mut self, - _cid: u32, - _port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } -} deno_core::extension!( fetch, @@ -109,38 +13,29 @@ deno_core::extension!( esm = ["src/runtime.js"], ); -// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`. +// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`, +// also applied to the residual lazy sources below. // -// Specialized to our snapshot's inputs. Of the eight deno_* extensions -// we register via `init()`, seven ship pre-built `.js` files -// in their `esm` lists (webidl/url/console/web/crypto/fetch/net) — only -// `deno_telemetry`'s `extension!` macro lists `.ts` files -// (`telemetry.ts`, `util.ts`), so the TypeScript branch is needed -// solely for that crate. Our local `fetch` extension contributes -// `src/runtime.js` (pure JS). No `node:` imports happen at snapshot -// build time, no `.mjs`, no user-supplied modules. So: +// Specialized to our inputs: the deno_* extensions ship `.js`, except +// `deno_telemetry` whose lazy scripts are `.ts`. Our local `fetch` extension +// contributes `src/runtime.js`. No `node:` imports, no `.mjs`, no +// user-supplied modules. So: // - `.js` → pass through. // - `.ts` → transpile via deno_ast (deno_telemetry only). // - anything else → build bug (deno shipping an unexpected file type // or us mislabelling one), panic loudly rather than emit a broken // snapshot. // -// No source maps: the snapshot is a binary blob the runtime loads — source -// maps would never be consumed. +// No source maps: nothing consumes them. // -// The signature still returns `Result<_, JsErrorBox>` because that's what -// `extension_transpiler` expects, but we never construct one — parse and -// transpile failures are build-time bugs in deno's own .ts internals (or -// in our runtime.js, if we ever change its extension), so they panic. +// The signature returns `Result<_, JsErrorBox>` because that's what +// `extension_transpiler` expects, but we never construct one: parse and +// transpile failures are build-time bugs, so they panic. // -// This replaces a call to `deno_runtime::transpile::maybe_transpile_source` -// from `deno_runtime 0.198.0`. The original is more general (handles -// `node:` modules, `.mjs`, emits source maps in debug builds, plumbs -// errors via `JsErrorBox`); none of that surface is reachable in our -// build. Dropping the `deno_runtime` dep eliminates a -// `deno_cache → rusqlite → libsqlite3-sys 0.35` transitive chain that -// collides with sqlx-sqlite's `libsqlite3-sys 0.30` (cargo's -// `links = "sqlite3"` rule). +// `deno_runtime::transpile::maybe_transpile_source` is the general version; +// depending on `deno_runtime` would pull a +// `deno_cache → rusqlite → libsqlite3-sys` chain that collides with +// sqlx-sqlite's `libsqlite3-sys` (cargo's `links = "sqlite3"` rule). fn maybe_transpile_source( name: ModuleName, source: ModuleCodeString, @@ -154,7 +49,7 @@ fn maybe_transpile_source( let parsed = deno_ast::parse_module(ParseParams { specifier: deno_core::url::Url::parse(&name).unwrap(), - text: source.into(), + text: source.as_str().into(), media_type, capture_tokens: false, scope_analysis: false, @@ -169,7 +64,13 @@ fn maybe_transpile_source( ..Default::default() }, &deno_ast::TranspileModuleOptions::default(), - &deno_ast::EmitOptions::default(), + // The default appends an inline source map comment, which is a + // syntax error where deno_core evaluates a lazy script as an + // expression. + &deno_ast::EmitOptions { + source_map: deno_ast::SourceMapOption::None, + ..Default::default() + }, ) .unwrap_or_else(|e| panic!("snapshot transpile: emit failed for {name}: {e}")) .into_source(); @@ -177,27 +78,66 @@ fn maybe_transpile_source( Ok((transpiled.text.into(), None)) } +/// One `(specifier, source)` table of the extension scripts the snapshot did +/// not evaluate. The snapshot only carries scripts that ran while it was built, +/// so a script first loaded in a live isolate has to come from here. +/// +/// deno_core binary-searches the table and reads each source as ASCII without +/// checking, so it must be sorted by specifier, free of duplicates, and ASCII. +fn residual_table( + name: &str, + files: impl Iterator, + consumed: &HashSet, +) -> String { + let mut files: Vec<_> = files.filter(|f| !consumed.contains(f.specifier)).collect(); + files.sort_by_key(|f| f.specifier); + files.dedup_by_key(|f| f.specifier); + + let mut out = format!("pub static {name}: &[(&str, &str)] = &[\n"); + for file in files { + let source = file + .load() + .unwrap_or_else(|e| panic!("could not load {}: {e}", file.specifier)); + let (code, _) = + maybe_transpile_source(ModuleName::from_static(file.specifier), source).unwrap(); + assert!( + code.as_str().is_ascii(), + "{} is not ASCII after transpiling", + file.specifier + ); + writeln!(out, " ({:?}, {:?}),", file.specifier, code.as_str()).unwrap(); + } + out.push_str("];\n"); + out +} + fn main() { println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); - let exts = vec![ - deno_telemetry::deno_telemetry::init(), - deno_webidl::deno_webidl::init(), - deno_url::deno_url::init(), - deno_console::deno_console::init(), - deno_web::deno_web::init::(Arc::new(BlobStore::default()), None), - deno_crypto::deno_crypto::init(None), - deno_fetch::deno_fetch::init::(Default::default()), - deno_net::deno_net::init::(None, None), + // Same list and order as `create_nativets_runtime`, which supplies their + // state through `lazy_init_extensions`. + let exts: Vec = vec![ + deno_telemetry::deno_telemetry::lazy_init(), + deno_webidl::deno_webidl::lazy_init(), + deno_web::deno_web::lazy_init(), + deno_crypto::deno_crypto::lazy_init(), + deno_fetch::deno_fetch::lazy_init(), + deno_net::deno_net::lazy_init(), fetch::init(), ]; - // Build the file path to the snapshot. + let lazy_js: Vec<_> = exts + .iter() + .flat_map(|e| e.lazy_loaded_js_files.iter().cloned()) + .collect(); + let lazy_esm: Vec<_> = exts + .iter() + .flat_map(|e| e.lazy_loaded_esm_files.iter().cloned()) + .collect(); + let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let snapshot_path = o.join("FETCH_SNAPSHOT.bin"); - // Create the snapshot. let output = deno_core::snapshot::create_snapshot( deno_core::snapshot::CreateSnapshotOptions { cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), @@ -213,9 +153,17 @@ fn main() { ) .unwrap(); - let mut file = std::fs::File::create(snapshot_path).unwrap(); + let mut file = std::fs::File::create(o.join("FETCH_SNAPSHOT.bin")).unwrap(); file.write_all(&output.output).unwrap(); + let consumed: HashSet = output.consumed_lazy_specifiers.into_iter().collect(); + let residual = format!( + "{}{}", + residual_table("RESIDUAL_LAZY_JS", lazy_js.into_iter(), &consumed), + residual_table("RESIDUAL_LAZY_ESM", lazy_esm.into_iter(), &consumed), + ); + std::fs::write(o.join("residual_lazy_sources.rs"), residual).unwrap(); + for path in output.files_loaded_during_snapshot { println!("cargo:rerun-if-changed={}", path.display()); } diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index 1c2a76cda5..750cdaa609 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -22,9 +22,7 @@ mod smoke_tests; mod cert_tests; use std::{ - borrow::Cow, cell::RefCell, - path::PathBuf, rc::Rc, sync::{Arc, LazyLock, Mutex}, }; @@ -32,17 +30,55 @@ use std::{ // Re-export deno_telemetry for use by windmill-worker's otel proxy pub use deno_telemetry; +/// The real environment and filesystem, except that the OTLP protocol always +/// reads as HTTP. +/// +/// The caller points `OTEL_EXPORTER_OTLP_ENDPOINT` at its own HTTP collector +/// only while [`init_telemetry`] runs, and the instance's own exporter shares +/// `OTEL_EXPORTER_OTLP_PROTOCOL`. Honouring `grpc` there would build an exporter +/// that reads the endpoint again on every export, and `console` one that never +/// reads it, so either would send job telemetry past that collector. +struct HttpOtlpSys; + +impl sys_traits::BaseEnvVar for HttpOtlpSys { + fn base_env_var_os(&self, key: &std::ffi::OsStr) -> Option { + if key == "OTEL_EXPORTER_OTLP_PROTOCOL" { + return Some("http/protobuf".into()); + } + sys_traits::impls::RealSys.base_env_var_os(key) + } +} + +impl sys_traits::BaseFsRead for HttpOtlpSys { + fn base_fs_read( + &self, + path: &std::path::Path, + ) -> std::io::Result> { + sys_traits::impls::RealSys.base_fs_read(path) + } +} + +/// `deno_telemetry::init` for the isolates of this process. +pub fn init_telemetry( + rt_config: deno_telemetry::OtelRuntimeConfig, + config: deno_telemetry::OtelConfig, +) -> anyhow::Result<()> { + deno_telemetry::init(&HttpOtlpSys, rt_config, config) +} + use deno_ast::ParseParams; use deno_core::{ + error::CoreErrorKind, op2, serde_v8, url, v8::{self, IsolateHandle}, Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, }; use deno_error::JsErrorBox; -use deno_fetch::FetchPermissions; -use deno_net::NetPermissions; +use deno_permissions::{ + Permissions, PermissionsContainer, PermissionsOptions, RuntimePermissionDescriptorParser, +}; use deno_tls::{rustls::pki_types::CertificateDer, rustls::RootCertStore, RootCertStoreProvider}; -use deno_web::{BlobStore, TimersPermission}; +use deno_web::{BlobStore, InMemoryBroadcastChannel}; use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; @@ -57,156 +93,55 @@ use windmill_common::worker::{write_file, Connection, WINDMILL_DIR}; // ── Snapshot-matched extensions ────────────────────────────────────── // -// `deno_core` 0.352 validates that the snapshot's extension list is a -// *prefix* of the runtime's extension list (snapshot does not need an -// exact match — runtime is allowed to add extensions at the tail, but -// must not reorder or omit any that the snapshot baked in). +// The snapshot (build.rs) is built from the deno_* extensions in their +// `lazy_init()` form followed by this local `fetch` ext. deno_core requires the +// runtime's extension list to start with that same list in that same order; it +// may append more, which is how the windmill `ext` carrying our own ops gets in. // -// Our snapshot (in build.rs) is the same eight deno_* extensions ending -// with this local `fetch` ext. The runtime adds one extra entry at the -// end — the windmill `ext` carrying our own ops — which is fine because -// it's after the snapshot prefix. -// -// This local `fetch` extension declaration must be present in both -// build.rs and lib.rs so the type passes through the `init()` macro. -// The ESM is already in the snapshot, so this `init()` call at runtime -// is a no-op for esm — the registration just records the ext. +// This declaration must exist in both build.rs and lib.rs. The ESM is already +// in the snapshot, so `init()` at runtime only records the extension. deno_core::extension!( fetch, esm_entry_point = "ext:fetch/src/runtime.js", esm = ["src/runtime.js"], ); -// ── Permission container ───────────────────────────────────────────── +include!(concat!(env!("OUT_DIR"), "/residual_lazy_sources.rs")); -/// `no_network` is what `//no_network` promises: no outbound traffic of any kind, -/// the Windmill API included (the client module reaches it through `fetch`). -/// Enforced here and by disabling the deno_net ops, never in JS: user code can -/// call `Deno.core.ops.*` directly and skip any JS-level guard. -pub struct PermissionsContainer { - pub no_network: bool, -} - -const NO_NETWORK_ANNOTATION: &str = "no_network"; +// ── Permissions ────────────────────────────────────────────────────── /// Every op deno_net registers, see `create_nativets_runtime`. Read from the /// extension itself so an upgrade that adds an op cannot leave it enabled. static NO_NETWORK_DISABLED_OPS: LazyLock> = LazyLock::new(|| { - deno_net::deno_net::init::(None, None) + deno_net::deno_net::lazy_init() .ops .iter() .map(|op| op.name) .collect() }); -impl PermissionsContainer { - fn deny_net(&self, target: &str) -> Result<(), deno_permissions::PermissionCheckError> { - if self.no_network { - Err(deno_permissions::PermissionDeniedError::Fatal { - access: format!( - "net access to {target} (the script is annotated //{NO_NETWORK_ANNOTATION})" - ), - } - .into()) - } else { - Ok(()) - } - } -} - -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("\"{}\"", url.host_str().unwrap_or_default())) - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - path: Cow<'a, std::path::Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - Ok(deno_fs::CheckedPath::Unresolved(path)) - } - - #[inline(always)] - fn check_write<'a>( - &mut self, - path: Cow<'a, std::path::Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - Ok(deno_fs::CheckedPath::Unresolved(path)) - } - - #[inline(always)] - fn check_net_vsock( - &mut self, - cid: u32, - port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("vsock {cid}:{port}")) - } -} - -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -impl NetPermissions for PermissionsContainer { - fn check_read( - &mut self, - p: &str, - _api_name: &str, - ) -> Result { - self.deny_net(&format!("unix socket \"{p}\""))?; - Ok(PathBuf::from(p)) - } - - fn check_write( - &mut self, - p: &str, - _api_name: &str, - ) -> Result { - self.deny_net(&format!("unix socket \"{p}\""))?; - Ok(PathBuf::from(p)) - } - - fn check_net>( - &mut self, - host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("\"{}\"", host.0.as_ref())) - } - - fn check_write_path<'a>( - &mut self, - p: Cow<'a, std::path::Path>, - _api_name: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - self.deny_net(&format!("unix socket \"{}\"", p.display()))?; - Ok(p) - } - - fn check_vsock( - &mut self, - cid: u32, - port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("vsock {cid}:{port}")) - } +/// What an isolate may do: reach the network, unless `no_network`, and nothing +/// else (no filesystem, environment, subprocess or FFI). Nothing prompts. +/// +/// `no_network` is what `//no_network` promises: no outbound traffic of any +/// kind, the Windmill API included (the client module reaches it through +/// `fetch`). Enforced here and by disabling the deno_net ops, never in JS: user +/// code can call `Deno.core.ops.*` directly and skip any JS-level guard. +fn isolate_permissions(no_network: bool) -> anyhow::Result { + let parser = Arc::new(RuntimePermissionDescriptorParser::new( + sys_traits::impls::RealSys, + )); + // An empty list means "all hosts", for a grant and for a denial alike. + let options = PermissionsOptions { + allow_net: (!no_network).then(Vec::new), + deny_net: no_network.then(Vec::new), + prompt: false, + ..Default::default() + }; + let permissions = Permissions::from_options(parser.as_ref(), &options) + .map_err(|e| anyhow::anyhow!("invalid isolate permissions: {e}"))?; + Ok(PermissionsContainer::new(parser, permissions)) } // ── Types ──────────────────────────────────────────────────────────── @@ -228,7 +163,7 @@ pub struct NativeAnnotation { /// script; `None` leaves the default in force. pub fetch_response_timeout_secs: Option, /// `//no_network`: the isolate may open no connection at all, see - /// [`PermissionsContainer`]. + /// [`isolate_permissions`]. pub no_network: bool, } @@ -413,13 +348,10 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> { // deno_fetch requires a TLS provider; install ring as default (idempotent). let _ = rustls::crypto::ring::default_provider().install_default(); - let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ - "--stack-size=1024".to_string(), - "--no-harmony-import-assertions".to_string(), - ]) - .into_iter() - .skip(1) - .collect::>(); + let unrecognized_v8_flags = deno_core::v8_set_flags(vec!["--stack-size=1024".to_string()]) + .into_iter() + .skip(1) + .collect::>(); if !unrecognized_v8_flags.is_empty() { init_err = Some(format!( @@ -435,7 +367,7 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> { // SIGSEGV in WasmCodePointerTable::AllocateUninitializedEntry() on x86_64 Linux. // See: https://github.com/denoland/deno_core/issues/952 let platform = deno_core::v8::new_unprotected_default_platform(0, false).make_shared(); - deno_core::JsRuntime::init_platform(Some(platform), false); + deno_core::JsRuntime::init_platform(Some(platform)); }); if let Some(msg) = init_err { @@ -666,17 +598,15 @@ pub(crate) fn create_nativets_runtime( ..Default::default() }; + // Same list and order as the snapshot (build.rs); their state follows in + // `lazy_init_extensions` below, which must match it too. let exts: Vec = vec![ - deno_telemetry::deno_telemetry::init(), - deno_webidl::deno_webidl::init(), - deno_url::deno_url::init(), - deno_console::deno_console::init(), - deno_web::deno_web::init::(Arc::new(BlobStore::default()), None), - // Registered after deno_web to keep the snapshot (build.rs) a prefix of - // the runtime extension list; deno_crypto declares deps = [deno_webidl, deno_web]. - deno_crypto::deno_crypto::init(None), - deno_fetch::deno_fetch::init::(fetch_options), - deno_net::deno_net::init::(None, None), + deno_telemetry::deno_telemetry::lazy_init(), + deno_webidl::deno_webidl::lazy_init(), + deno_web::deno_web::lazy_init(), + deno_crypto::deno_crypto::lazy_init(), + deno_fetch::deno_fetch::lazy_init(), + deno_net::deno_net::lazy_init(), fetch::init(), ext, ]; @@ -688,6 +618,8 @@ pub(crate) fn create_nativets_runtime( deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128), ), startup_snapshot: Some(RUNTIME_SNAPSHOT), + residual_lazy_js_sources: RESIDUAL_LAZY_JS, + residual_lazy_esm_sources: RESIDUAL_LAZY_ESM, module_loader: Some(Rc::new(deno_core::FsModuleLoader)), extension_transpiler: None, ..Default::default() @@ -714,12 +646,28 @@ pub(crate) fn create_nativets_runtime( y * 2 }); + js_runtime + .lazy_init_extensions(vec![ + deno_telemetry::deno_telemetry::args(), + deno_webidl::deno_webidl::args(), + deno_web::deno_web::args( + Arc::new(BlobStore::default()), + None, + false, + InMemoryBroadcastChannel::default(), + ), + deno_crypto::deno_crypto::args(None), + deno_fetch::deno_fetch::args(fetch_options), + deno_net::deno_net::args(None, None), + ]) + .map_err(windmill_common::error::to_anyhow)?; + let (log_sender, log_receiver) = mpsc::unbounded_channel::(); { let op_state = js_runtime.op_state(); let mut op_state = op_state.borrow_mut(); - op_state.put(PermissionsContainer { no_network }); + op_state.put(isolate_permissions(no_network)?); op_state.put(MainArgs { args: initial_args }); op_state.put(LogString { s: log_sender }); } @@ -731,7 +679,7 @@ pub(crate) fn create_nativets_runtime( .execute_script( "", format!( - "globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms} }})" + "globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms}, noNetwork: {no_network} }})" ), ) .map_err(windmill_common::error::to_anyhow)?; @@ -777,7 +725,7 @@ pub(crate) fn extract_global_string( js_runtime: &mut JsRuntime, global: v8::Global, ) -> Result, String> { - let scope = &mut js_runtime.handle_scope(); + deno_core::scope!(scope, js_runtime); let local = v8::Local::new(scope, global); match serde_v8::from_v8::>(scope, local) { Ok(s) => Ok(unsafe_raw(s.unwrap_or_else(|| "null".to_string()))), @@ -929,8 +877,8 @@ pub async fn eval_fetch_timeout( for line in logged.as_deref().unwrap_or(&log).lines() { tracing::info!( target: "windmill:job_log", - job_id = ?job_id, - workspace_id = ?w_id_for_tracing, + job_id = %job_id, + workspace_id = %w_id_for_tracing, "{line}" ); if *OTEL_JOB_LOGS { @@ -1169,7 +1117,10 @@ import("file:///eval.ts").then((module) => module.{main_fn}(...args)) Ok(global) => { extract_global_string(js_runtime, global).map_err(|e| ExecuteError::Script(e)) } - Err(deno_core::error::CoreError::Js(e)) => { + Err(e) if matches!(e.as_kind(), CoreErrorKind::Js(_)) => { + let CoreErrorKind::Js(e) = e.into_kind() else { + unreachable!("matched as a JS error above") + }; let source = e.frames.first().and_then(|f| { f.file_name .as_ref() diff --git a/backend/windmill-runtime-nativets/src/runtime.js b/backend/windmill-runtime-nativets/src/runtime.js index 88e10f765a..86d20dd022 100644 --- a/backend/windmill-runtime-nativets/src/runtime.js +++ b/backend/windmill-runtime-nativets/src/runtime.js @@ -1,39 +1,42 @@ -import * as abortSignal from "ext:deno_web/03_abort_signal.js"; -import * as domException from "ext:deno_web/01_dom_exception.js"; -import * as base64 from "ext:deno_web/05_base64.js"; -import * as console from "ext:deno_console/01_console.js"; -import * as encoding from "ext:deno_web/08_text_encoding.js"; -import * as event from "ext:deno_web/02_event.js"; -import * as fetch from "ext:deno_fetch/26_fetch.js"; -import * as file from "ext:deno_web/09_file.js"; -import * as fileReader from "ext:deno_web/10_filereader.js"; -import * as formData from "ext:deno_fetch/21_formdata.js"; -import * as headers from "ext:deno_fetch/20_headers.js"; -import * as streams from "ext:deno_web/06_streams.js"; -import * as timers from "ext:deno_web/02_timers.js"; -import * as url from "ext:deno_url/00_url.js"; -import * as net from "ext:deno_net/01_net.js"; -import * as tls from "ext:deno_net/02_tls.js"; -import * as urlPattern from "ext:deno_url/01_urlpattern.js"; -import * as webidl from "ext:deno_webidl/00_webidl.js"; -import * as crypto from "ext:deno_crypto/00_crypto.js"; -import * as response from "ext:deno_fetch/23_response.js"; -import * as request from "ext:deno_fetch/23_request.js"; -import "ext:deno_web/02_structured_clone.js"; -import * as globalInterfaces from "ext:deno_web/04_global_interfaces.js"; -// Namespace imports (not side-effect-only) so their constructors are reachable -// for the globalThis wiring below. The module bodies still execute on -// evaluation, so their side effects apply. -import * as messagePort from "ext:deno_web/13_message_port.js"; -import * as compression from "ext:deno_web/14_compression.js"; -import * as performance from "ext:deno_web/15_performance.js"; -import "ext:deno_web/16_image_data.js"; -import "ext:deno_fetch/27_eventsource.js"; +import { core, internals } from "ext:core/mod.js"; + +// deno_crypto brands CryptoKeys with this symbol and reads it from `internals` +// when it evaluates; in Deno a deno_node module sets it, which is not loaded here. +internals.kKeyObject ??= Symbol("kKeyObject"); + +// Every script loaded here runs at snapshot-build time and is baked into the +// snapshot. One that is first loaded at run time comes from the residual table +// build.rs emits instead. +const abortSignal = core.loadExtScript("ext:deno_web/03_abort_signal.js"); +const domException = core.loadExtScript("ext:deno_web/01_dom_exception.js"); +const base64 = core.loadExtScript("ext:deno_web/05_base64.js"); +const console = core.loadExtScript("ext:deno_web/01_console.js"); +const encoding = core.loadExtScript("ext:deno_web/08_text_encoding.js"); +const event = core.loadExtScript("ext:deno_web/02_event.js"); +const fetch = core.loadExtScript("ext:deno_fetch/26_fetch.js"); +const file = core.loadExtScript("ext:deno_web/09_file.js"); +const fileReader = core.loadExtScript("ext:deno_web/10_filereader.js"); +const formData = core.loadExtScript("ext:deno_fetch/21_formdata.js"); +const headers = core.loadExtScript("ext:deno_fetch/20_headers.js"); +const streams = core.loadExtScript("ext:deno_web/06_streams.js"); +const timers = core.loadExtScript("ext:deno_web/02_timers.js"); +const url = core.loadExtScript("ext:deno_web/00_url.js"); +const urlPattern = core.loadExtScript("ext:deno_web/01_urlpattern.js"); +const webidl = core.loadExtScript("ext:deno_webidl/00_webidl.js"); +const crypto = core.loadExtScript("ext:deno_crypto/00_crypto.js"); +const response = core.loadExtScript("ext:deno_fetch/23_response.js"); +const request = core.loadExtScript("ext:deno_fetch/23_request.js"); +const globalInterfaces = core.loadExtScript( + "ext:deno_web/04_global_interfaces.js", +); +const messagePort = core.loadExtScript("ext:deno_web/13_message_port.js"); +const compression = core.loadExtScript("ext:deno_web/14_compression.js"); +const performance = core.loadExtScript("ext:deno_web/15_performance.js"); // deno_fetch applies no deadline, so a peer that accepts a request and then // never answers leaves `await fetch(...)` pending until the job timeout, which // self-hosted defaults to 7 days. -const ORIGINAL_FETCH = fetch.fetch; +const DENO_FETCH = fetch.fetch; // deno_web's timers reject any `this` other than undefined/globalThis, so // `timers.setTimeout(...)` passes the module namespace and throws "Illegal @@ -50,6 +53,14 @@ const abortControllerAbort = Function.prototype.call.bind( ); const ReflectApply = Reflect.apply; +// Forwards the original argument count, like every caller below relies on. The +// extra promise hop under `no_network` costs the same-tick settling of an +// aborted fetch, which only matters where a request could have been made. +function ORIGINAL_FETCH() { + const p = ReflectApply(DENO_FETCH, undefined, arguments); + return noNetwork ? promiseThen(p, undefined, rewordNoNetworkDenial) : p; +} + // Installed per isolate by __wmInitPerIsolate; 0 disables. Only a backstop // for the impossible case of fetch running before that init. let fetchResponseTimeoutMs = 900_000; @@ -74,6 +85,21 @@ function fetchResponseTimeoutError(requestUrl, timeoutMs) { ); } +// Set per isolate for a `//no_network` script. The denial itself is the +// isolate's permissions (Rust); this only replaces deno's "run again with the +// --allow-net flag" hint, which names a flag a script author cannot pass. +let noNetwork = false; + +function rewordNoNetworkDenial(e) { + if (e?.name === "NotCapable" && typeof e.message === "string") { + e.message = e.message.replace( + ", run again with the --allow-net flag", + ": the script is annotated //no_network", + ); + } + throw e; +} + globalThis.atob = base64.atob; globalThis.btoa = base64.btoa; // Not `async`, for the same reason deno_fetch's own outer fetch isn't: WPT @@ -152,7 +178,22 @@ globalThis.console = new console.Console((msg, level) => ); globalThis.AbortController = abortSignal.AbortController; globalThis.AbortSignal = abortSignal.AbortSignal; -globalThis.crypto = crypto.crypto; +// A getter, not a value: `crypto.crypto` mints a cppgc object, and the cppgc +// heap is not attached while this module runs at snapshot-build time. +Object.defineProperty(globalThis, "crypto", { + configurable: true, + enumerable: true, + get: () => crypto.crypto, + // Assignment has to keep working, as it does on a plain property. + set(value) { + Object.defineProperty(globalThis, "crypto", { + configurable: true, + enumerable: true, + writable: true, + value, + }); + }, +}); globalThis.Crypto = crypto.Crypto; globalThis.CryptoKey = crypto.CryptoKey; globalThis.SubtleCrypto = crypto.SubtleCrypto; @@ -233,6 +274,7 @@ globalThis.__wmInitPerIsolate = (config) => { if (config != null && typeof config.fetchResponseTimeoutMs === "number") { fetchResponseTimeoutMs = config.fetchResponseTimeoutMs; } + noNetwork = config?.noNetwork === true; // setTimeOrigin() seeds performance.timeOrigin from the isolate's wall clock; // without it timeOrigin is undefined and `timeOrigin + performance.now()` is NaN. @@ -256,13 +298,14 @@ globalThis.__wmInitPerIsolate = (config) => { }; // Expose bootstrapOtel globally so it can be called from Rust after runtime creation. -// We use dynamic import so deno_telemetry isn't loaded during snapshot creation. +// Loaded on first call so deno_telemetry isn't evaluated during snapshot creation. // Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] // consoleConfig: 0=ignore, 1=capture, 2=replace globalThis.__bootstrapOtel = () => { - import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => { - bootstrap([1, 0, 1, 0]); - // Expose enterSpan for setting parent trace context - globalThis.__enterSpan = enterSpan; - }); + const { bootstrap, enterSpan } = core.loadExtScript( + "ext:deno_telemetry/telemetry.ts", + ); + bootstrap([1, 0, 1, 0]); + // Expose enterSpan for setting parent trace context + globalThis.__enterSpan = enterSpan; }; diff --git a/backend/windmill-runtime-nativets/src/smoke_tests.rs b/backend/windmill-runtime-nativets/src/smoke_tests.rs index f3da460227..1373a8abd4 100644 --- a/backend/windmill-runtime-nativets/src/smoke_tests.rs +++ b/backend/windmill-runtime-nativets/src/smoke_tests.rs @@ -451,6 +451,10 @@ export async function main(): Promise<{ uuid: string; nonzero: boolean; sha256: .map((b) => b.toString(16).padStart(2, "0")) .join(""); + // `crypto` is installed as an accessor; polyfill-style assignment must + // still work rather than throw for want of a setter. + (globalThis as any).crypto = globalThis.crypto; + return { uuid, nonzero, sha256 }; } "#; diff --git a/backend/windmill-runtime-nativets/tests/no_network.rs b/backend/windmill-runtime-nativets/tests/no_network.rs index 876207fc26..dc7086f22e 100644 --- a/backend/windmill-runtime-nativets/tests/no_network.rs +++ b/backend/windmill-runtime-nativets/tests/no_network.rs @@ -55,6 +55,9 @@ async function attempt(f) {{ export async function main() {{ return {{ fetch: await attempt(() => fetch("http://127.0.0.1:{port}/").then((r) => r.text())), + data_url: await attempt(() => fetch("data:text/plain,ok").then((r) => r.text())), + blob_url: await attempt(() => fetch(URL.createObjectURL(new Blob(["ok"]))).then((r) => r.text())), + raw_fetch: await attempt(() => Deno.core.ops.op_fetch("GET", "http://127.0.0.1:{port}/", [], null, false, null, null)), tcp: await attempt(() => Deno.core.ops.op_net_connect_tcp({{ hostname: "127.0.0.1", port: {port} }}, null, null)), unix: await attempt(() => Deno.core.ops.op_net_connect_unix("/var/run/docker.sock")), quic: await attempt(() => Deno.core.ops.op_quic_endpoint_create({{ hostname: "localhost", port: 0 }}, true)), @@ -76,6 +79,15 @@ async fn no_network_denies_fetch_and_raw_socket_ops() { fetch.contains("no_network"), "fetch was not denied: {fetch}" ); + // Neither opens a connection, so neither is network access. + assert_eq!(out["data_url"], "allowed", "{out}"); + assert_eq!(out["blob_url"], "allowed", "{out}"); + // The global `fetch` only words the refusal; the op under it is what refuses. + let raw_fetch = out["raw_fetch"].as_str().unwrap(); + assert!( + raw_fetch.contains("Requires net access"), + "the fetch op was not denied: {raw_fetch}" + ); for key in ["tcp", "unix", "quic"] { let msg = out[key].as_str().unwrap(); assert!( diff --git a/backend/windmill-runtime-nativets/tests/otel_e2e.rs b/backend/windmill-runtime-nativets/tests/otel_e2e.rs index 22fd18de90..d1853c461d 100644 --- a/backend/windmill-runtime-nativets/tests/otel_e2e.rs +++ b/backend/windmill-runtime-nativets/tests/otel_e2e.rs @@ -71,7 +71,7 @@ async fn spawn_capturing_http(captured: Arc>>>) -> u16 { /// drifts away from `tracing_enabled + Capture`, this test breaks /// before the customer-facing panic does. fn init_with_ee_otel_config() { - deno_telemetry::init( + windmill_runtime_nativets::init_telemetry( deno_telemetry::OtelRuntimeConfig { runtime_name: "windmill-nativets".into(), runtime_version: "0".into(), @@ -157,13 +157,12 @@ export async function main(): Promise {{ "fetch target should have been hit exactly once" ); - // 4. Force the BatchSpanProcessor to flush so the exporter posts - // to our mock collector synchronously (default flush interval - // is ~5s; tests can't wait that long). - deno_telemetry::flush(); - // Exporter is async over the OTel runtime; give it a beat to - // actually send the HTTP request. - tokio::time::sleep(Duration::from_millis(500)).await; + // 4. The BatchSpanProcessor exports on its own schedule (about 5s by + // default) and deno_telemetry exposes no flush, so wait for it. + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while otlp_hits.lock().await.is_empty() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } let otlp_captured = otlp_hits.lock().await; assert!( diff --git a/backend/windmill-runtime-nativets/tests/otel_init.rs b/backend/windmill-runtime-nativets/tests/otel_init.rs index fdf1ac944d..80ef8683aa 100644 --- a/backend/windmill-runtime-nativets/tests/otel_init.rs +++ b/backend/windmill-runtime-nativets/tests/otel_init.rs @@ -46,7 +46,7 @@ async fn ee_otel_init_config_populates_otel_globals() { // Step 1: reproduce the footgun. `OtelConfig::default()` has // tracing/metrics off and `console = Ignore`, which trips the // 0.31 early-return — Ok(()) but OnceCell stays empty. - deno_telemetry::init( + windmill_runtime_nativets::init_telemetry( deno_telemetry::OtelRuntimeConfig { runtime_name: "windmill-nativets-test".into(), runtime_version: "0".into(), @@ -63,10 +63,17 @@ async fn ee_otel_init_config_populates_otel_globals() { tracing_enabled/console fields." ); + // The instance's own exporter shares this variable and may be set to gRPC, + // or to anything else. Native telemetry must not read it: a value + // deno_telemetry rejects outright shows it is never consulted. + // SAFETY: the only test in this binary, so nothing reads the environment + // concurrently. + unsafe { std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", "not-a-protocol") }; + // Step 2: re-call init with the exact config shape that the EE // `load_internal_otel_exporter` ships — this is what production // hits, so the test exercises the actual prod call path. - deno_telemetry::init( + windmill_runtime_nativets::init_telemetry( deno_telemetry::OtelRuntimeConfig { runtime_name: "windmill-nativets".into(), runtime_version: "0".into(), diff --git a/backend/windmill-store/src/variables.rs b/backend/windmill-store/src/variables.rs index 8f288dc85e..151b75bd85 100644 --- a/backend/windmill-store/src/variables.rs +++ b/backend/windmill-store/src/variables.rs @@ -563,7 +563,8 @@ async fn check_path_conflict(db: &DB, w_id: &str, path: &str) -> Result<()> { /// /// The check is purely structural and never decrypts, so it cannot act as a /// decryption/padding oracle for a caller who can write but not read secrets. -/// `encrypt` (AES-256-CBC) always yields standard base64 decoding to a non-zero +/// `encrypt` (AES-256-GCM, or legacy AES-256-CBC; see `windmill_common::workspace_crypt`) +/// always yields standard base64 decoding to a non-zero /// multiple of the 16-byte block size; anything else cannot be our ciphertext. /// Values stored by an external backend ($vault:/$aws_sm:/$azure_kv: markers) /// are not workspace ciphertext and are passed through untouched. diff --git a/backend/windmill-worker-volumes/src/lib.rs b/backend/windmill-worker-volumes/src/lib.rs index e2e4ab866b..364795510a 100644 --- a/backend/windmill-worker-volumes/src/lib.rs +++ b/backend/windmill-worker-volumes/src/lib.rs @@ -54,11 +54,45 @@ pub struct VolumeMount { pub target: String, } +/// A file's size and last change time. Sync-back trusts a pulled file's stored hash only while +/// its stamp is the one taken at hand-off: size alone misses a rewrite of the same length. A +/// write within the filesystem's timestamp granularity of the hand-off is not seen. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FileStamp { + size: u64, + changed: Option<(i64, i64)>, +} + +impl FileStamp { + /// Whether the file is as it was at `earlier`. An unreadable change time never matches. + pub fn unchanged_since(&self, earlier: &FileStamp) -> bool { + self.changed.is_some() && self == earlier + } + + pub fn of(meta: &std::fs::Metadata) -> Self { + // ctime rather than mtime: a tool that restores mtime (cp -p, tar, rsync -t) cannot set it. + #[cfg(unix)] + let changed = { + use std::os::unix::fs::MetadataExt; + Some((meta.ctime(), meta.ctime_nsec())) + }; + #[cfg(not(unix))] + let changed = meta + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| (d.as_secs() as i64, d.subsec_nanos() as i64)); + Self { size: meta.len(), changed } + } +} + pub struct VolumeState { pub mount: VolumeMount, pub local_dir: PathBuf, pub manifest: HashMap, pub symlinks: HashMap, + /// Stamp of each pulled file as the job received it, keyed like `manifest`. + pub stamps: HashMap, } pub struct DownloadStats { diff --git a/backend/windmill-worker/nsjail/build.csharp.config.proto b/backend/windmill-worker/nsjail/build.csharp.config.proto new file mode 100644 index 0000000000..a926d923f2 --- /dev/null +++ b/backend/windmill-worker/nsjail/build.csharp.config.proto @@ -0,0 +1,128 @@ +name: "csharp restore/build" + +mode: ONCE +hostname: "csharp" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "{DOTNET_ROOT}" + dst: "{DOTNET_ROOT}" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# project files, nuget.config, and the published output dir are all here; cwd of dotnet +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# DOTNET_CLI_HOME + NUGET_PACKAGES (nuget/). Writable, like the install/run jails' +# shared caches. +mount { + src: "{CSHARP_CACHE_DIR}" + dst: "{CSHARP_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +envar: "HOME=/tmp" + +#{DEV} diff --git a/backend/windmill-worker/nsjail/lock.php.config.proto b/backend/windmill-worker/nsjail/lock.php.config.proto new file mode 100644 index 0000000000..d06c1699a6 --- /dev/null +++ b/backend/windmill-worker/nsjail/lock.php.config.proto @@ -0,0 +1,124 @@ +name: "php composer install" + +mode: ONCE +hostname: "php" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# composer.json/composer.lock live here and vendor/ is written here; cwd of composer +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# COMPOSER_HOME: package download cache. Writable, like the install/run jails' +# shared caches. +mount { + src: "{COMPOSER_CACHE_DIR}" + dst: "{COMPOSER_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +{CA_MOUNT} + +envar: "HOME=/tmp" + +#{DEV} diff --git a/backend/windmill-worker/nsjail/lock.py.config.proto b/backend/windmill-worker/nsjail/lock.py.config.proto new file mode 100644 index 0000000000..14119c4089 --- /dev/null +++ b/backend/windmill-worker/nsjail/lock.py.config.proto @@ -0,0 +1,133 @@ +name: "python lock compile" + +mode: ONCE +hostname: "python" +log_level: ERROR +time_limit: {TIMEOUT} + +# Dependency resolution may invoke the PEP 517 build backend of an sdist whose +# metadata is not static, which runs arbitrary package code (e.g. setup.py). +# That build, and uv's bytecode compile pool, size thread/fd pools to the host's +# CPU count, so these mirror download.py.config.proto rather than the run sandbox. +rlimit_as: 8192 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 10000 + +envar: "HOME=/tmp" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: true +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# requirements.in -> requirements.txt, cwd of uv +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# shared uv cache: resolution metadata + where sdist build backends run +# (builds-v0/). Writable, as the install jail's target already is; a poisoned +# shared cache is the known residual, not a new one. +mount { + src: "{UV_CACHE_DIR}" + dst: "{UV_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{PY_INSTALL_DIR}" + dst: "{PY_INSTALL_DIR}" + is_bind: true +} + +{INDEX_CERT_MOUNT} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +#{DEV} diff --git a/backend/windmill-worker/src/ai/stream_event_processor.rs b/backend/windmill-worker/src/ai/stream_event_processor.rs index fe8a3b9b00..c4d17f080a 100644 --- a/backend/windmill-worker/src/ai/stream_event_processor.rs +++ b/backend/windmill-worker/src/ai/stream_event_processor.rs @@ -1,3 +1,8 @@ +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + use async_trait::async_trait; use windmill_ai::{query_builder::StreamEventSink, types::*}; use windmill_common::{error::Error, worker::Connection}; @@ -10,11 +15,13 @@ use crate::job_logger::append_result_stream; pub struct StreamEventProcessor { tx: Option>, pub handle: Option>, + /// Shared by every clone, so a sink lent to a parser counts into its owner. + streamed: Arc, } impl Clone for StreamEventProcessor { fn clone(&self) -> Self { - Self { tx: self.tx.clone(), handle: None } + Self { tx: self.tx.clone(), handle: None, streamed: self.streamed.clone() } } } @@ -48,13 +55,19 @@ impl StreamEventProcessor { } }); - Self { tx: Some(tx), handle: Some(handle) } + Self { tx: Some(tx), handle: Some(handle), streamed: Arc::default() } } /// Create a silent StreamEventProcessor that only accumulates events locally /// without persisting them to the database. Used when streaming is disabled. pub fn new_silent() -> Self { - Self { tx: None, handle: None } + Self { tx: None, handle: None, streamed: Arc::default() } + } + + /// How many events have been streamed to the job's readers. A silent processor + /// streams none. + pub fn streamed_events(&self) -> usize { + self.streamed.load(Ordering::Relaxed) } pub fn to_handle(self) -> Option> { @@ -79,6 +92,7 @@ impl StreamEventSink for StreamEventProcessor { Ok(event_json) => { let event_json = format!("{}\n", event_json); events_str.push_str(&event_json); + self.streamed.fetch_add(1, Ordering::Relaxed); if let Err(err) = tx .send(event_json.clone()) diff --git a/backend/windmill-worker/src/ai/utils.rs b/backend/windmill-worker/src/ai/utils.rs index 7437184385..f0a0b5c20f 100644 --- a/backend/windmill-worker/src/ai/utils.rs +++ b/backend/windmill-worker/src/ai/utils.rs @@ -103,9 +103,9 @@ pub fn filter_schema_by_input_transforms( .get_mut("properties") .and_then(|p| p.as_object_mut()) { - for key in &keys_to_remove { - properties.remove(key); - } + // `remove` swaps the last entry into the freed slot, so removing in set order + // would leave the survivors in a different order on every run. + properties.retain(|key, _| !keys_to_remove.contains(key)); } // Also remove from required array @@ -570,7 +570,7 @@ pub async fn load_mcp_tools( // is not allowed to read (resources:read:{path}). Reading through the raw // db pool here would bypass the authorization enforced by the regular MCP // tools API (get_mcp_tools) and act as a confused deputy. - let mcp_resource = client + let mut mcp_resource = client .get_resource_value::(path) .await .map_err(|e| { @@ -615,6 +615,18 @@ pub async fn load_mcp_tools( None }; + // Header secrets go through the same permissioned path as the token. + for (name, value) in mcp_resource.headers.iter_mut().flatten() { + if let Some(var_path) = value.strip_prefix("$var:") { + *value = client.get_variable_value(var_path).await.map_err(|e| { + Error::internal_err(format!( + "Failed to resolve variable {} for header {} of MCP resource {}: {}", + var_path, name, resource_name, e + )) + })?; + } + } + // Create new MCP client for this execution tracing::debug!("Creating fresh MCP client for {}", resource_name); let mcp_conn = McpClient::from_resource(mcp_resource, token) @@ -736,3 +748,32 @@ pub fn any_tool_needs_previous_result(tools: &[Tool]) -> bool { false }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// A tool's schema sits in the prompt prefix the provider caches on, so the + /// parameters left for the model must keep their order on every run. + #[test] + fn filtering_preset_inputs_keeps_the_remaining_property_order() { + let keys = ["a", "b", "c", "d", "e", "f", "g", "h"]; + let props = keys + .iter() + .map(|k| format!(r#""{k}":{{"type":"string"}}"#)) + .collect::>() + .join(","); + let schema = + RawValue::from_string(format!(r#"{{"type":"object","properties":{{{props}}}}}"#)) + .unwrap(); + let preset = ["a", "c", "f"] + .iter() + .map(|k| (k.to_string(), InputTransform::Static { value: to_raw_value(&1) })) + .collect(); + + let filtered = filter_schema_by_input_transforms(schema, &preset).unwrap(); + let value: serde_json::Value = serde_json::from_str(filtered.get()).unwrap(); + let remaining: Vec<&String> = value["properties"].as_object().unwrap().keys().collect(); + assert_eq!(remaining, ["b", "d", "e", "g", "h"]); + } +} diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 7dd2382731..1b889aea34 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -43,6 +43,7 @@ use windmill_ai::{ common_outbound_headers, needs_unavailable_oauth_exchange, retain_effective_credentials, }, query_builder::{BuildRequestArgs, ParsedResponse}, + retry::{as_transient, into_final_error, response_error, send_error, Backoff, MAX_RETRIES}, types::*, utils::{pinned_ai_client_for, should_use_structured_output_tool}, }; @@ -1659,10 +1660,17 @@ pub async fn run_agent( .await; let mut retried_context = false; + let mut backoff = Backoff::default(); + let streamed_events = || { + stream_event_processor + .as_ref() + .map_or(0, |processor| processor.streamed_events()) + }; let (parsed, request_message_count) = loop { // How many messages this request carries, so the provider's prompt count can // later be told apart from what the response and its tool results add. let request_message_count = messages.context().len(); + let streamed_before = streamed_events(); // Handle AWS Bedrock provider specially using the official SDK let attempt: error::Result<_> = async { @@ -1818,7 +1826,7 @@ pub async fn run_agent( let resp = build_http_request(&endpoint, &auth_headers, request_body) .send() .await - .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; + .map_err(|e| send_error(e, "Failed to call API"))?; match resp.error_for_status_ref() { Ok(_) => { @@ -1829,6 +1837,7 @@ pub async fn run_agent( } Err(e) => { let status = resp.status(); + let headers = resp.headers().clone(); let text = resp .text() .await @@ -1898,10 +1907,9 @@ pub async fn run_agent( query_builder = create_chat_completions_query_builder(&credentials); include_usage = true; } else { - return Err(Error::internal_err(format!( - "API error calling {}: {} - {}", - endpoint, e, text - ))); + let message = + format!("API error calling {}: {} - {}", endpoint, e, text); + return Err(response_error(status, &headers, &text, message)); } } } @@ -1961,7 +1969,45 @@ pub async fn run_agent( return Err(error); } } - Err(error) => return Err(error), + Err(error) => { + let Some(transient) = as_transient(&error) else { + return Err(error); + }; + // Readers have no way to take back what they were shown, so a call + // that already streamed part of its answer fails as it is. + if streamed_events() != streamed_before { + append_logs( + &job.id, + &job.workspace_id, + "AI provider call failed after part of its answer had streamed; not retried.\n".to_string(), + conn, + ) + .await; + return Err(into_final_error(error)); + } + let Some(delay) = backoff.next_delay(transient) else { + return Err(into_final_error(error)); + }; + append_logs( + &job.id, + &job.workspace_id, + format!( + "AI provider call failed: {}. Retrying in {:.1}s (retry {}/{MAX_RETRIES}).\n", + transient.message, + delay.as_secs_f64(), + backoff.retries() + ), + conn, + ) + .await; + let mut cancelled = cancel_rx.clone(); + tokio::select! { + _ = tokio::time::sleep(delay) => {} + Ok(_) = cancelled.wait_for(|cancelled| *cancelled) => { + return Err(Error::ExecutionErr("Job cancelled".to_string())); + } + } + } } }; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index bb47d3e4be..de33a2b9e6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -3273,9 +3273,9 @@ pub async fn handle_wac_v2_output( .or(if own_runnable { None } else { job.priority }), None, // authed false, // running - None, // end_user_email - None, // trigger - None, // suspended_mode + job.permissioned_as_end_user_email.clone(), + None, // trigger + None, // suspended_mode job.job_token_scopes.as_deref(), ) .await?; @@ -3731,7 +3731,10 @@ pub async fn handle_wac_v2_output( // status row, the order every child completion takes. let segment_ms = sqlx::query_scalar!( "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) - UPDATE v2_job_queue q SET running = false, started_at = null + UPDATE v2_job_queue q SET running = false, started_at = null, + extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras + ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at)) + END FROM prev WHERE q.id = $1 RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", job.id, @@ -4083,6 +4086,7 @@ async fn handle_dedicated_bunnative( result, result_columns: None, mem_peak: 0, + resource_usage: None, canceled_by: None, success, cached_res_path: None, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 01523eba59..6088fb4960 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -643,6 +643,7 @@ pub async fn update_worker_ping_for_failed_init_script( wm_memory_usage: None, job_isolation: None, native_mode: None, + draining: None, ping_type: PingType::InitScript, }, ) @@ -1725,6 +1726,23 @@ pub async fn cached_result_path( hasher.update(b"wac_step:"); hasher.update(step_key.as_bytes()); } + // What a run returns can depend on who it ran for (`WM_END_USER_EMAIL`, a data table used + // as the viewer), so a result cached for one viewer is never served to another. Read from + // `job_perms`: a flow job re-fetched between steps does not carry it. + let end_user: Option> = sqlx::query_scalar( + "SELECT end_user_email FROM job_perms WHERE job_id = $1 AND workspace_id = $2", + ) + .bind(job.id) + .bind(&job.workspace_id) + .fetch_optional(db) + .await?; + // Length-prefixed: the arguments follow undelimited, and the end of one viewer's address + // must not read as the start of another run's first argument name. + if let Some(end_user) = end_user.flatten() { + hasher.update(b"end_user:"); + hasher.update((end_user.len() as u64).to_le_bytes()); + hasher.update(end_user.as_bytes()); + } hash_args( db, client, @@ -1961,6 +1979,18 @@ pub async fn save_in_cache( CACHED_RESULTS.insert(cached_path, Arc::new(store_cache_resource)); } +pub(crate) fn deno_not_installed_error() -> Error { + Error::ExecutionErr(format!( + "Deno is not installed on this worker (looked for it at {}). The standard Windmill images \ + no longer ship Deno: official support for it on those images was discontinued after Deno \ + announced the end of development of the Deno runtime (https://deno.com/blog/cloudflare). \ + We recommend migrating this script to Bun. To keep running Deno scripts, use the \ + windmill-full or windmill-ee-full image on every worker that accepts the deno tag, or \ + install Deno on them and point DENO_PATH at it.", + crate::DENO_PATH.as_str() + )) +} + fn tentatively_improve_error(err: Error, executable: &str) -> Error { #[cfg(unix)] let err_msgs = vec!["os error 2", "os error 3", "No such file or directory"]; @@ -1969,6 +1999,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { let err_msgs = vec!["program not found", "os error 2", "os error 3"]; if err_msgs.iter().any(|msg| err.to_string().contains(msg)) { + // An absolute DENO_PATH is checked before spawning, where a missing job dir + // cannot be mistaken for a missing binary. + if executable == crate::DENO_PATH.as_str() && !Path::new(executable).is_absolute() { + return deno_not_installed_error(); + } return Error::internal_err(format!( "Executable {executable} not found on worker. PATH: {}", *PATH_ENV @@ -2457,6 +2492,52 @@ mod tests { } } + // A cache hit returns before the job runs, so the key is all that keeps a result computed + // for one viewer (a data table used as the viewer, `WM_END_USER_EMAIL`) from another. + #[sqlx::test(migrations = "../migrations")] + async fn a_cached_result_is_keyed_by_the_recorded_viewer(pool: sqlx::PgPool) { + let client = test_client(); + + // The last two differ only in where the viewer's address ends and the first argument + // name starts. + let mut paths = vec![]; + for (viewer, arg) in [ + (Some("a@windmill.dev"), "mode"), + (Some("a@windmill.dev"), "mode"), + (Some("b@windmill.dev"), "mode"), + (None, "mode"), + (Some("a@windmill.co"), "mode"), + (Some("a@windmill.com"), "ode"), + ] { + let args = HashMap::from([(arg.to_string(), to_raw_value(&"all"))]); + let job = MiniPulledJob { + id: uuid::Uuid::new_v4(), + args: Some(sqlx::types::Json(args)), + ..test_job() + }; + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, + workspace_id, groups, folders, end_user_email) + VALUES ($1, 'publisher@windmill.dev', 'publisher', false, false, 'test', '{}', + '{}', $2)", + ) + .bind(job.id) + .bind(viewer) + .execute(&pool) + .await + .unwrap(); + paths.push(cached_result_path(&pool, &client, &job, None).await.unwrap()); + } + assert_eq!( + paths[0], paths[1], + "one viewer shares their own cached result" + ); + assert_ne!(paths[0], paths[2], "two viewers share a cached result"); + assert_ne!(paths[0], paths[3]); + assert_ne!(paths[2], paths[3]); + assert_ne!(paths[4], paths[5], "an address runs into the argument names"); + } + #[tokio::test] async fn test_transform_array_over_1000_passthrough() { let db_url = std::env::var("DATABASE_URL") diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index d393171f7f..7d77f8703f 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -33,7 +33,7 @@ use crate::{ get_proxy_envs_for_lang, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry_with_workspace_override, CSHARP_CACHE_DIR, - DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, + DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; #[cfg(feature = "csharp")] @@ -47,6 +47,9 @@ use crate::SYSTEM_ROOT; #[cfg(feature = "csharp")] const NSJAIL_CONFIG_RUN_CSHARP_CONTENT: &str = include_str!("../nsjail/run.csharp.config.proto"); +#[cfg(feature = "csharp")] +const NSJAIL_CONFIG_BUILD_CSHARP_CONTENT: &str = + include_str!("../nsjail/build.csharp.config.proto"); #[cfg(feature = "csharp")] #[cfg(windows)] @@ -124,6 +127,9 @@ pub async fn generate_nuget_lockfile( gen_cs_proj(code, job_dir, reqs, lines_to_remove)?; + // `dotnet restore` resolves and downloads packages but does not import their + // MSBuild `build/*.targets`, so it runs no package code — only `dotnet publish` + // (jailed below) does — so it runs unsandboxed. let mut gen_lockfile_cmd = Command::new(DOTNET_PATH.as_str()); gen_lockfile_cmd .current_dir(job_dir) @@ -384,31 +390,82 @@ async fn build_cs_proj( } } - let mut build_cs_cmd = Command::new(DOTNET_PATH.as_str()); - build_cs_cmd - .current_dir(job_dir) - .env_clear() - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) - .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") - .env("DOTNET_NOLOGO", "true") - .env("MSBUILDDISABLENODEREUSE", "1") - .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) - .args(vec![ - "publish", - "--configuration", - "Release", - "-o", - &format!("{job_dir}/out"), - "--no-self-contained", - "-p:PublishSingleFile=true", - "-p:IncludeNativeLibrariesForSelfExtract=true", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + let out_dir = format!("{job_dir}/out"); + let publish_args = vec![ + "publish", + "--configuration", + "Release", + "-o", + out_dir.as_str(), + "--no-self-contained", + "-p:PublishSingleFile=true", + "-p:IncludeNativeLibrariesForSelfExtract=true", + ]; + // `dotnet publish` imports and runs the MSBuild targets/tasks of restored NuGet + // packages (a package's build/*.targets execute at build, not at restore), i.e. + // arbitrary package code. Confine it under nsjail when sandboxing is on, the same + // way the run step is; HOME is set to /tmp by the config, keep_env passes the + // explicit dotnet/proxy vars, and the job dir + shared nuget cache are writable. + // Without sandboxing, publish runs directly with no isolation. + // mut is required by the #[cfg(windows)] env block below; unused on unix. + #[allow(unused_mut)] + let mut build_cs_cmd = if is_sandboxing_enabled() { + std::fs::create_dir_all(&*CSHARP_CACHE_DIR)?; + let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await; + write_file( + job_dir, + "build.config.proto", + NSJAIL_CONFIG_BUILD_CSHARP_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{JOB_DIR}", job_dir) + .replace("{CSHARP_CACHE_DIR}", &*CSHARP_CACHE_DIR) + .replace("{DOTNET_ROOT}", DOTNET_ROOT.as_str()) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .as_str(), + )?; + let mut nsjail_args = vec!["--config", "build.config.proto", "--", DOTNET_PATH.as_str()]; + nsjail_args.extend(publish_args.iter().copied()); + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) + .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") + .env("DOTNET_NOLOGO", "true") + .env("MSBUILDDISABLENODEREUSE", "1") + .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) + .envs(PROXY_ENVS.clone()) + .envs(&*crate::worker::WHITELIST_ENVS) + .args(nsjail_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + cmd + } else { + let mut cmd = Command::new(DOTNET_PATH.as_str()); + cmd.current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) + .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") + .env("DOTNET_NOLOGO", "true") + .env("MSBUILDDISABLENODEREUSE", "1") + .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) + .args(publish_args.clone()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + cmd + }; #[cfg(windows)] build_cs_cmd @@ -433,14 +490,20 @@ async fn build_cs_proj( .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), ); - let build_cs_process = start_child_process(build_cs_cmd, DOTNET_PATH.as_str(), true).await?; + let build_cs_executable = if is_sandboxing_enabled() { + NSJAIL_PATH.as_str() + } else { + DOTNET_PATH.as_str() + }; + let build_cs_process = start_child_process(build_cs_cmd, build_cs_executable, true).await?; handle_child( job_id, conn, mem_peak, canceled_by, build_cs_process, - false, + // jailed when sandboxing is on, so mem sampling finds the real process + is_sandboxing_enabled(), worker_name, w_id, "dotnet publish", diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 2c5c17bb27..58388c3cfd 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,7 +7,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + build_command_with_isolation, create_args_and_out_file, deno_not_installed_error, + get_reserved_variables, parse_npm_config, read_file, read_result, start_child_process, OccupancyMetrics, StreamNotifier, }, @@ -155,6 +156,17 @@ async fn get_common_deno_proc_envs( return deno_envs; } +/// A spawn failure only names Deno when Deno itself is the spawned executable; behind an +/// isolation wrapper or a dedicated worker it surfaces as an opaque exit. A bare `DENO_PATH` +/// is resolved through PATH at spawn time, so it is left to the spawn error. +fn ensure_deno_installed() -> Result<()> { + let path = std::path::Path::new(DENO_PATH.as_str()); + if path.is_absolute() && !path.exists() { + return Err(deno_not_installed_error()); + } + Ok(()) +} + pub async fn generate_deno_lock( job_id: &Uuid, code: &str, @@ -167,6 +179,7 @@ pub async fn generate_deno_lock( base_internal_url: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result { + ensure_deno_installed()?; let _ = write_file(job_dir, "main.ts", code)?; let import_map_path = format!("{job_dir}/import_map.json"); @@ -249,6 +262,7 @@ pub async fn handle_deno_job( occupancy_metrics: &mut OccupancyMetrics, has_stream: &mut bool, ) -> error::Result> { + ensure_deno_installed()?; let annotations = TypeScriptAnnotations::parse(inner_content); // let mut start = Instant::now(); @@ -745,6 +759,7 @@ pub async fn start_worker( use crate::common::build_envs_map; + ensure_deno_installed()?; let _ = write_file(job_dir, "main.ts", inner_content)?; let common_deno_proc_envs = get_common_deno_proc_envs( &token, diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index b8c850a2f4..58e47b9332 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -2758,6 +2758,7 @@ async fn transform_attach_datatable( DatatableAccess::PermissionedAs { permissioned_as: &job.permissioned_as, email: &job.permissioned_as_email, + end_user_email: job.permissioned_as_end_user_email.as_deref(), }, ) .await? diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 22a2bf4e8b..31eb8034f9 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -59,6 +59,56 @@ use windmill_common::tracing_init::{OTEL_JOB_LOGS, OTEL_PREFIX, QUIET_MODE, VERB lazy_static::lazy_static! { pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); + // A job runs several children in turn (install, build, run): their CPU time adds up + // here until the job completes. + static ref JOB_CPU_TIME_MS: std::sync::Mutex> = Default::default(); +} + +/// CPU time of the children run so far for the job, which stops being tracked. +pub(crate) fn take_job_cpu_time_ms(job_id: &Uuid) -> Option { + JOB_CPU_TIME_MS.lock().unwrap().remove(job_id) +} + +/// CPU time in ms from the content of `/proc//stat`: the process's own user and +/// system time plus that of the children it waited for, which is where the time of a +/// job run under a wrapper (nsjail, unshare) ends up. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn parse_proc_stat_cpu_ms(stat: &str) -> Option { + // The values are in USER_HZ ticks, which Linux fixes at 100 per second. + const MS_PER_TICK: i64 = 10; + // `comm` is free text in parentheses: the numbered fields resume after its last `)`, + // with the state as field 3 and utime, stime, cutime, cstime as fields 14 to 17. + let fields: Vec<&str> = stat.rsplit_once(')')?.1.split_ascii_whitespace().collect(); + let ticks = fields + .get(11..15)? + .iter() + .map(|f| f.parse::().ok()) + .sum::>()?; + Some(ticks * MS_PER_TICK) +} + +/// Resolves once the child has exited, to its CPU time. The child is left un-reaped: +/// `/proc//stat` is only readable until then, so the caller reaps afterwards. +#[cfg(target_os = "linux")] +async fn wait_for_exit_cpu_time_ms(pid: u32) -> Option { + use nix::sys::wait::{waitid, Id, WaitPidFlag}; + let (tx, rx) = tokio::sync::oneshot::channel(); + // A plain thread rather than `spawn_blocking`: the wait cannot be cancelled, and a + // blocking task still parked on a live child would hold up the runtime's shutdown. + std::thread::Builder::new() + .name("job-cpu-time".to_string()) + .spawn(move || { + let cpu = waitid( + Id::Pid(Pid::from_raw(pid as i32)), + WaitPidFlag::WEXITED | WaitPidFlag::WNOWAIT, + ) + .ok() + .and_then(|_| std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()) + .and_then(|stat| parse_proc_stat_cpu_ms(&stat)); + let _ = tx.send(cpu); + }) + .ok()?; + rx.await.ok().flatten() } // - kill windows process along with all child processes @@ -210,6 +260,53 @@ pub async fn handle_child( } } + impl KillReason { + /// Reports a kill once the process is gone: a span event on the enclosing span and, + /// through the logs bridge, a log record carrying the job's log context. + fn emit_canceled_event( + &self, + job_id: &Uuid, + timeout_reason: &str, + signal: &'static str, + detected_at: chrono::DateTime, + ) { + // `force` is only known for a soft cancel: a job completed under the worker + // may have been force canceled or completed by anything else server-side. + let (kill_reason, canceled_by, canceled_reason, force) = match self { + KillReason::TooManyLogs => return, + KillReason::Timeout { .. } => { + ("timeout", Some("timeout"), Some(timeout_reason), None) + } + KillReason::Cancelled(by) => ( + "cancelled", + by.as_ref().and_then(|x| x.username.as_deref()), + by.as_ref().and_then(|x| x.reason.as_deref()), + Some(false), + ), + KillReason::AlreadyCompleted => ("already_completed", None, None, None), + }; + let exited_at = chrono::Utc::now(); + let root_job = windmill_common::log_context::current_log_context() + .and_then(|c| c.root_job.clone()); + let rfc3339 = |t: chrono::DateTime| { + t.to_rfc3339_opts(chrono::SecondsFormat::Millis, true) + }; + tracing::info!( + job_id = %job_id, + root_job, + kill_reason, + canceled_by, + canceled_reason, + force, + kill_signal = signal, + cancel_detected_at = %rfc3339(detected_at), + process_exited_at = %rfc3339(exited_at), + kill_duration_ms = (exited_at - detected_at).num_milliseconds(), + "job.canceled" + ); + } + } + let (timeout_duration, timeout_warn_msg, timeout_source) = resolve_job_timeout(&conn, w_id, job_id, custom_timeout).await; if let Some(msg) = timeout_warn_msg { @@ -220,7 +317,15 @@ pub async fn handle_child( let wait_on_child = async { let kill_reason = tokio::select! { biased; - result = Box::into_pin(child.wait()) => return result.map(Ok), + result = async { + #[cfg(target_os = "linux")] + if let Some(pid) = pid.filter(|_| job_id != Uuid::nil()) { + if let Some(cpu) = wait_for_exit_cpu_time_ms(pid).await { + *JOB_CPU_TIME_MS.lock().unwrap().entry(job_id).or_insert(0) += cpu; + } + } + Box::into_pin(child.wait()).await + } => return result.map(Ok), Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs, _ = sleep(timeout_duration) => KillReason::Timeout { phase: child_name.to_string(), @@ -232,6 +337,7 @@ pub async fn handle_child( UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted, }, }; + let detected_at = chrono::Utc::now(); tx.send(()).expect("rx should never be dropped"); drop(tx); @@ -268,59 +374,78 @@ pub async fn handle_child( } }; - #[allow(unused_variables)] - if let Some(id) = child.id() { - if *MAX_WAIT_FOR_SIGINT > 0 { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); + // Resolves once the process is reaped, to the last signal it was sent, or to `None` + // when the kill could not be confirmed. SIGINT and SIGTERM are only sent where the + // sends below are compiled in: elsewhere an exit during their wait is unsignaled. + let unix_signal = |s: &'static str| { + if cfg!(any(target_os = "linux", target_os = "macos")) { s } else { "none" } + }; + let kill = async { + #[allow(unused_variables)] + if let Some(id) = child.id() { + if *MAX_WAIT_FOR_SIGINT > 0 { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); - for _ in 0..*MAX_WAIT_FOR_SIGINT { + for _ in 0..*MAX_WAIT_FOR_SIGINT { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } if child.try_wait().is_ok_and(|x| x.is_some()) { - break; + set_reason.await; + return Ok(Some(unix_signal("SIGINT"))); } - sleep(Duration::from_secs(1)).await; } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); + if sigterm { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); + + for _ in 0..*MAX_WAIT_FOR_SIGTERM { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } + if child.try_wait().is_ok_and(|x| x.is_some()) { + set_reason.await; + return Ok(Some(unix_signal("SIGTERM"))); + } } } - if sigterm { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); - - for _ in 0..*MAX_WAIT_FOR_SIGTERM { - if child.try_wait().is_ok_and(|x| x.is_some()) { - break; + #[cfg(windows)] + { + let pid_to_kill = child.id(); + let killed = match kill_process_tree(pid_to_kill).await { + Ok(_) => { + tracing::debug!( + "successfully killed process tree with PID: {:?}", + pid_to_kill + ); + true } - sleep(Duration::from_secs(1)).await; - } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); - } + Err(e) => { + tracing::error!("failed to kill process tree: {:?}", e); + false + } + }; + set_reason.await; + return Ok(killed.then_some("taskkill")); } - } - #[cfg(windows)] - { - let pid_to_kill = child.id(); - match kill_process_tree(pid_to_kill).await { - Ok(_) => tracing::debug!( - "successfully killed process tree with PID: {:?}", - pid_to_kill - ), - Err(e) => tracing::error!("failed to kill process tree: {:?}", e), - }; - set_reason.await; - return Ok(Err(kill_reason)); - } - #[cfg(unix)] - { - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await; - kill.map(|()| Err(kill_reason)) + #[cfg(unix)] + { + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await; + kill.map(|()| Some("SIGKILL")) + } + }; + let signal: io::Result> = kill.await; + if let Some(signal) = signal? { + kill_reason.emit_canceled_event(&job_id, &timeout_reason, signal, detected_at); } + Ok(Err(kill_reason)) }; let mut stream_result = Vec::new(); @@ -957,9 +1082,9 @@ where let conn = conn.clone(); // No tick at t=0: a job that finishes within the first interval issues none of the - // ping/metric statements below. Cancels are picked up by the first job ping (500 ms in, - // 2 s on agent workers). Memory is still sampled at start, without SQL: it is the only - // `mem_peak` reading a short job gets, and the completed job row reports it. + // ping/metric statements below. Cancels are picked up by the next job ping. Memory is still + // sampled at start, without SQL: it is the only `mem_peak` reading a short job gets, + // and the completed job row reports it. let mut interval = interval_at(Instant::now() + update_job_interval, update_job_interval); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); *mem_peak = (*mem_peak).max(get_mem.next().await.unwrap_or(0)); @@ -982,6 +1107,7 @@ where let wm_memory_usage = get_windmill_memory_usage(); tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024); let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics()); + crate::resource_metrics::record_worker_resources(worker_name, memory_usage, wm_memory_usage, occupancy.as_ref()); if job_id != Uuid::nil() { if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await { tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); @@ -1023,11 +1149,10 @@ where } } } - if matches!(conn, Connection::Http(_)) { - if i % 4 != 0 { - // only ping every 4th time (2s) on http agent mode - continue; - } + // An agent worker pays an HTTP round trip per ping, so it pings every 2 s for + // the first 10 s, then every 5 s. + if matches!(conn, Connection::Http(_)) && i % if i < 20 { 4 } else { 10 } != 0 { + continue; } let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| { tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); @@ -1074,8 +1199,8 @@ fn child_joined_output_stream( .take() .expect("child did not have a handle to stdout"); - let stdout = BufReader::new(stdout).lines(); - let stderr = BufReader::new(stderr).lines(); + let stdout = BufReader::new(stdout).split(b'\n'); + let stderr = BufReader::new(stderr).split(b'\n'); stream::select( lines_to_stream(stderr, true, job_id.clone(), w_id.clone()) .map(|l| l.map(|line| OutputLine { stderr: true, line })), @@ -1085,18 +1210,32 @@ fn child_joined_output_stream( } pub fn lines_to_stream( - mut lines: tokio::io::Lines, + mut lines: tokio::io::Split, stderr: bool, job_id: Uuid, w_id: String, ) -> impl futures::Stream> { stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) - .poll_next_line(cx) - .map(|result| process_streaming_log_lines(result, stderr, &job_id, &w_id)) + .poll_next_segment(cx) + .map(|result| { + let result = result.map(|segment| segment.map(decode_output_line)); + process_streaming_log_lines(result, stderr, &job_id, &w_id) + }) }) } +/// Child output is not guaranteed to be UTF-8 (e.g. pwsh on Windows writes in the +/// console code page), so decode lossily: a strict decode errors out and ends log +/// capture for the rest of the job. Strips a trailing `\r` so CRLF output doesn't keep it. +pub(crate) fn decode_output_line(mut segment: Vec) -> String { + if segment.last() == Some(&b'\r') { + segment.pop(); + } + String::from_utf8(segment) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) +} + pub fn process_status( program: &str, status: ExitStatus, @@ -1133,6 +1272,21 @@ pub fn process_status( mod tests { use super::*; + #[test] + fn proc_stat_cpu_time_survives_a_hostile_process_name() { + // utime 150, stime 50, cutime 700, cstime 100 ticks: 1000 ticks, 10 s. + let tail = "Z 1 42 42 0 -1 4194560 100 200 0 0 150 50 700 100 20 0 1 0 12345 0 0"; + assert_eq!( + parse_proc_stat_cpu_ms(&format!("42 (bun) {tail}")), + Some(10_000) + ); + assert_eq!( + parse_proc_stat_cpu_ms(&format!("42 (a) R 9 (9 9 9 9 9 9 9 9 9) {tail}")), + Some(10_000) + ); + assert_eq!(parse_proc_stat_cpu_ms("42 (bun) Z 1 42"), None); + } + // A non-positive timeout means "unset" to `resolve_job_timeout`, so an // exhausted budget that reported 0 would hand the next phase no deadline at // all — the opposite of what running out of time should do. @@ -1143,4 +1297,33 @@ mod tests { let left = JobDeadline(Some(Instant::now() + Duration::from_secs(120))); assert!(matches!(left.remaining_secs(), Some(s) if (118..=120).contains(&s))); } + + // "Prüfe" in CP850/1252 is not UTF-8. A strict decode errors on that line and + // would stop log capture, dropping every line after it. + #[tokio::test] + async fn a_non_utf8_line_is_decoded_lossily_and_later_lines_still_arrive() { + let output: &[u8] = + b"start\r\nPr\x81fe Benutzer\nPr\xfcfe\nGr\xc3\xbc\xc3\x9fe\r\nafter\n\nlast"; + let lines: Vec = lines_to_stream( + BufReader::new(output).split(b'\n'), + false, + Uuid::nil(), + "w".into(), + ) + .map(|line| line.expect("reading from a byte slice cannot fail")) + .collect() + .await; + assert_eq!( + lines, + vec![ + "start", + "Pr\u{FFFD}fe Benutzer", + "Pr\u{FFFD}fe", + "Grüße", + "after", + "", + "last", + ] + ); + } } diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 6059936af6..aa1d9ab209 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -81,6 +81,7 @@ mod prepare_deps; mod python_executor; #[cfg(feature = "python")] mod python_versions; +mod resource_metrics; pub mod result_processor; #[cfg(feature = "rust")] mod rust_executor; diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index aac7d549db..6309ba8c95 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -1229,6 +1229,7 @@ pub async fn do_postgresql( DatatableAccess::PermissionedAs { permissioned_as: &job.permissioned_as, email: &job.permissioned_as_email, + end_user_email: job.permissioned_as_end_user_email.as_deref(), }, ) .await? diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 03328b82cb..8378455ef4 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -17,6 +17,7 @@ use windmill_queue::MiniPulledJob; use windmill_parser::{MainArgSignature, Typ}; use windmill_queue::{append_logs, CanceledBy}; +use crate::common::DEV_CONF_NSJAIL; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, @@ -24,11 +25,13 @@ use crate::{ resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, + PHP_PATH, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, }; use windmill_common::client::AuthedClient; const NSJAIL_CONFIG_RUN_PHP_CONTENT: &str = include_str!("../nsjail/run.php.config.proto"); +const NSJAIL_CONFIG_LOCK_PHP_CONTENT: &str = include_str!("../nsjail/lock.php.config.proto"); lazy_static::lazy_static! { static ref RE: Regex = Regex::new(r"^//\s?(\S+)\s*$").unwrap(); @@ -199,15 +202,99 @@ pub async fn composer_install( write_file(job_dir, "composer.lock", lock)?; } - let mut child_cmd = Command::new(&*COMPOSER_PATH); - let args = vec!["install", "--no-dev", "--no-progress"]; - child_cmd - .current_dir(job_dir) - .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, &*COMPOSER_PATH, false).await?; + // `composer install` runs the plugin and script code of installed packages, so + // when sandboxing is on it is confined under nsjail like the run step. Inside the + // jail the environment is cleared down to what composer needs: PATH, COMPOSER_HOME + // (pointed at the mounted cache dir), the proxies, the operator-controlled + // WHITELIST_ENVS, and every other COMPOSER_* var from the worker env — Windmill has + // no managed composer-registry setting, so operators supply private Packagist auth + // and composer tuning (COMPOSER_AUTH, COMPOSER_PROCESS_TIMEOUT, …) through those. + // COMPOSER_CACHE_DIR is dropped so it can't point composer outside the jail, and a + // custom COMPOSER_CAFILE/COMPOSER_CAPATH is bind-mounted so TLS to private feeds + // keeps working. COMPOSER_PATH is Windmill-internal. Without sandboxing composer + // runs directly, with no isolation. + let composer_args = ["install", "--no-dev", "--no-progress"]; + let child_cmd = if is_sandboxing_enabled() { + // The cache dir is bind-mounted into the jail, so it must exist before + // nsjail builds the mount tree. + std::fs::create_dir_all(&*COMPOSER_CACHE_DIR)?; + let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await; + // A custom CA bundle/dir lives outside the jail's mounts; bind it in so + // composer's TLS can read it (mandatory: false → skipped when the path is unset). + let ca_mount = ["COMPOSER_CAFILE", "COMPOSER_CAPATH"] + .iter() + .filter_map(|k| std::env::var(k).ok().filter(|v| !v.trim().is_empty())) + .map(|p| { + format!( + "mount {{\n src: \"{p}\"\n dst: \"{p}\"\n is_bind: true\n mandatory: false\n}}\n" + ) + }) + .collect::(); + write_file( + job_dir, + "lock.config.proto", + NSJAIL_CONFIG_LOCK_PHP_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{JOB_DIR}", job_dir) + .replace("{COMPOSER_CACHE_DIR}", &*COMPOSER_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("{CA_MOUNT}", &ca_mount) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .as_str(), + )?; + + let composer_passthrough_envs: Vec<(String, String)> = std::env::vars() + .filter(|(k, _)| { + k.starts_with("COMPOSER_") + && k != "COMPOSER_HOME" + && k != "COMPOSER_CACHE_DIR" + && k != "COMPOSER_PATH" + }) + .collect(); + + let mut nsjail_args = vec![ + "--config", + "lock.config.proto", + "--", + COMPOSER_PATH.as_str(), + ]; + nsjail_args.extend(composer_args); + + let mut child_cmd = Command::new(NSJAIL_PATH.as_str()); + child_cmd + .current_dir(job_dir) + .env_clear() + // HOME is set to /tmp by the nsjail config; keep_env passes the rest. + .env("PATH", PATH_ENV.as_str()) + .envs(PROXY_ENVS.clone()) + .envs(&*crate::worker::WHITELIST_ENVS) + .envs(composer_passthrough_envs) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .args(nsjail_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + child_cmd + } else { + let mut child_cmd = Command::new(&*COMPOSER_PATH); + child_cmd + .current_dir(job_dir) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .args(composer_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + child_cmd + }; + let child_executable = if is_sandboxing_enabled() { + NSJAIL_PATH.as_str() + } else { + COMPOSER_PATH.as_str() + }; + let child_process = start_child_process(child_cmd, child_executable, false).await?; handle_child( job_id, @@ -215,7 +302,8 @@ pub async fn composer_install( mem_peak, canceled_by, child_process, - false, + // jailed when sandboxing is on, so mem sampling finds the real process + is_sandboxing_enabled(), worker_name, w_id, "composer install", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3fce0fb938..7f2291099a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -116,6 +116,7 @@ async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver, + index_url: Option, + /// Caches filled through these URLs must not be shared with other workspaces. + ws_scoped: bool, +} + +impl PipIndexes { + async fn snapshot(w_id: &str) -> Self { + let overrides = crate::workspace_registry_overrides(w_id).await; + let ws_value = |key: &str| match overrides.get(key) { + Some(serde_json::Value::String(s)) => Some(s.clone()), + _ => None, + }; + // An empty value means unset, and an empty workspace override unsets the global one. + Self { + extra_index_url: ws_value("pip_extra_index_url") + .or(PIP_EXTRA_INDEX_URL.read().await.clone()) + .filter(|s| !s.trim().is_empty()), + index_url: ws_value("pip_index_url") + .or(PIP_INDEX_URL.read().await.clone()) + .filter(|s| !s.trim().is_empty()), + ws_scoped: !overrides.is_empty(), + } + } + + /// The `(extra index, index)` URLs to hand uv. Kept apart from [`Self::snapshot`]: it runs + /// `EPHEMERAL_TOKEN_CMD`, which a caller served from a cache has no reason to pay for. + async fn urls( + self, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + ) -> (Option, Option) { + ( + read_ee_registry( + self.extra_index_url, + "pip extra index url", + job_id, + w_id, + conn, + ) + .await + .map(handle_ephemeral_token), + read_ee_registry(self.index_url, "pip index url", job_id, w_id, conn) + .await + .map(handle_ephemeral_token), + ) + } +} + lazy_static::lazy_static! { static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap(); @@ -2527,28 +2634,18 @@ pub async fn handle_python_reqs( job_id ); - let pip_indexes = ( - read_ee_registry_with_workspace_override( - PIP_EXTRA_INDEX_URL.read().await.clone(), - "pip_extra_index_url", - "pip extra index url", - job_id, - w_id, - conn, - ) - .await - .map(handle_ephemeral_token), - read_ee_registry_with_workspace_override( - PIP_INDEX_URL.read().await.clone(), - "pip_index_url", - "pip index url", - job_id, - w_id, - conn, - ) - .await - .map(handle_ephemeral_token), - ); + let pip_indexes = PipIndexes::snapshot(w_id).await; + let ws_scoped = pip_indexes.ws_scoped; + let pip_indexes = pip_indexes.urls(job_id, w_id, conn).await; + // The wheel dir is shared by every workspace on the worker, and by every worker through + // the object store, under a name that says nothing of the index it was filled from. A + // workspace with its own registries gets its own, as its resolution cache does. The id + // is hashed: it is not restricted to characters safe in a file name. + let ws_dir_suffix = if ws_scoped { + format!("-ws-{}", &calculate_hash(w_id)[..16]) + } else { + String::new() + }; // Cached paths let mut req_with_penv: Vec<(String, String)> = vec![]; @@ -2577,7 +2674,7 @@ pub async fn handle_python_reqs( let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( - "{py_prefix}/{}", + "{py_prefix}/{}{ws_dir_suffix}", req.replace(' ', "").replace('/', "").replace(':', "") ); if metadata(venv_p.clone() + "/.valid.windmill").await.is_ok() { diff --git a/backend/windmill-worker/src/resource_metrics.rs b/backend/windmill-worker/src/resource_metrics.rs new file mode 100644 index 0000000000..53c7dc5b4d --- /dev/null +++ b/backend/windmill-worker/src/resource_metrics.rs @@ -0,0 +1,235 @@ +use std::sync::{atomic::Ordering, Mutex, OnceLock}; + +use windmill_common::{ + otel_oss::{ + otel_add_worker_cpu, otel_record_job_memory_peak, otel_set_worker_memory, + otel_set_worker_occupancy_rate, + }, + worker::{get_cpu_stat, get_memory, CpuStat, WORKER_GROUP}, + OTEL_METRICS_ENABLED, +}; + +use crate::common::OccupancyResult; + +#[cfg(feature = "prometheus")] +struct PromMetrics { + memory_usage: prometheus::IntGaugeVec, + windmill_memory_usage: prometheus::IntGaugeVec, + memory_limit: prometheus::IntGaugeVec, + cpu_usage: prometheus::CounterVec, + cpu_periods: prometheus::IntCounterVec, + cpu_throttled_periods: prometheus::IntCounterVec, + cpu_throttled_seconds: prometheus::CounterVec, + occupancy_rate: prometheus::GaugeVec, + job_memory_peak: prometheus::HistogramVec, +} + +#[cfg(feature = "prometheus")] +lazy_static::lazy_static! { + static ref PROM_METRICS: Option = + if windmill_common::METRICS_ENABLED.load(Ordering::Relaxed) { + const WORKER: &[&str] = &["name", "worker_group"]; + Some(PromMetrics { + memory_usage: prometheus::register_int_gauge_vec!( + "worker_memory_usage_bytes", + "Memory used by the worker's container, page cache excluded", + WORKER + ) + .unwrap(), + windmill_memory_usage: prometheus::register_int_gauge_vec!( + "worker_windmill_memory_usage_bytes", + "Resident memory of the worker process itself", + WORKER + ) + .unwrap(), + memory_limit: prometheus::register_int_gauge_vec!( + "worker_memory_limit_bytes", + "Memory limit of the worker's container", + WORKER + ) + .unwrap(), + cpu_usage: prometheus::register_counter_vec!( + "worker_cpu_usage_seconds_total", + "CPU time consumed by the worker's container", + WORKER + ) + .unwrap(), + cpu_periods: prometheus::register_int_counter_vec!( + "worker_cpu_periods_total", + "CPU enforcement periods elapsed for the worker's container", + WORKER + ) + .unwrap(), + cpu_throttled_periods: prometheus::register_int_counter_vec!( + "worker_cpu_throttled_periods_total", + "CPU enforcement periods in which the worker's container was throttled", + WORKER + ) + .unwrap(), + cpu_throttled_seconds: prometheus::register_counter_vec!( + "worker_cpu_throttled_seconds_total", + "Time the worker's container spent throttled by its CPU limit", + WORKER + ) + .unwrap(), + occupancy_rate: prometheus::register_gauge_vec!( + "worker_occupancy_rate", + "Share of time the worker spent executing jobs, over the window", + &["name", "worker_group", "window"] + ) + .unwrap(), + job_memory_peak: prometheus::register_histogram_vec!( + "worker_job_memory_peak_bytes", + "Peak memory of a job's main process", + &["tag"], + windmill_common::worker::JOB_MEMORY_PEAK_BUCKETS.to_vec() + ) + .unwrap(), + }) + } else { + None + }; +} + +fn metrics_enabled() -> bool { + #[cfg(feature = "prometheus")] + if PROM_METRICS.is_some() { + return true; + } + OTEL_METRICS_ENABLED.load(Ordering::Relaxed) +} + +// Memory and CPU are read from the cgroup, which every worker of a process shares +// (native mode runs several). Only the first worker to report exports them, so that +// summing the series over workers does not count one container several times. +static CONTAINER_SERIES_OWNER: OnceLock = OnceLock::new(); +static MEMORY_LIMIT: Mutex>> = Mutex::new(None); + +/// Replaces the exported memory limit, for the ping that re-reads the cgroup limits. +pub(crate) fn set_memory_limit(limit: Option) { + *MEMORY_LIMIT.lock().unwrap() = Some(limit); +} +static LAST_CPU_STAT: Mutex = + Mutex::new(CpuStat { usage_usec: 0, nr_periods: 0, nr_throttled: 0, throttled_usec: 0 }); + +/// Exports the worker's resource readings. Meant for the ping ticks, which have already +/// read memory usage and occupancy; the only extra work is the cgroup `cpu.stat` read. +pub(crate) fn record_worker_resources( + worker_name: &str, + memory_usage: Option, + wm_memory_usage: Option, + occupancy: Option<&OccupancyResult>, +) { + if !metrics_enabled() { + return; + } + let worker_group = WORKER_GROUP.as_str(); + + if let Some(occupancy) = occupancy { + let windows = [ + ("total", Some(occupancy.occupancy_rate)), + ("15s", occupancy.occupancy_rate_15s), + ("5m", occupancy.occupancy_rate_5m), + ("30m", occupancy.occupancy_rate_30m), + ]; + for (window, rate) in windows { + let Some(rate) = rate.filter(|r| r.is_finite()) else { + continue; + }; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.occupancy_rate + .with_label_values(&[worker_name, worker_group, window]) + .set(rate as f64); + } + otel_set_worker_occupancy_rate(worker_name, worker_group, window, rate as f64); + } + } + + if CONTAINER_SERIES_OWNER.get_or_init(|| worker_name.to_string()) != worker_name { + return; + } + + let memory_limit = *MEMORY_LIMIT.lock().unwrap().get_or_insert_with(get_memory); + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + if let Some(v) = memory_usage { + m.memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = wm_memory_usage { + m.windmill_memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = memory_limit { + m.memory_limit.with_label_values(labels).set(v); + } + } + otel_set_worker_memory( + worker_name, + worker_group, + memory_usage, + wm_memory_usage, + memory_limit, + ); + + let Some(stat) = get_cpu_stat() else { + return; + }; + let delta = { + let mut last = LAST_CPU_STAT.lock().unwrap(); + // A reading below the previous one means the cgroup changed under the process: + // restart from it rather than exporting a negative increase. + let delta = if stat.usage_usec < last.usage_usec { + CpuStat::default() + } else { + CpuStat { + usage_usec: stat.usage_usec - last.usage_usec, + nr_periods: stat.nr_periods.saturating_sub(last.nr_periods), + nr_throttled: stat.nr_throttled.saturating_sub(last.nr_throttled), + throttled_usec: stat.throttled_usec.saturating_sub(last.throttled_usec), + } + }; + *last = stat; + delta + }; + let usage_secs = delta.usage_usec as f64 / 1e6; + let throttled_secs = delta.throttled_usec as f64 / 1e6; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + m.cpu_usage.with_label_values(labels).inc_by(usage_secs); + m.cpu_periods + .with_label_values(labels) + .inc_by(delta.nr_periods); + m.cpu_throttled_periods + .with_label_values(labels) + .inc_by(delta.nr_throttled); + m.cpu_throttled_seconds + .with_label_values(labels) + .inc_by(throttled_secs); + } + otel_add_worker_cpu( + worker_name, + worker_group, + usage_secs, + delta.nr_periods, + delta.nr_throttled, + throttled_secs, + ); +} + +/// `mem_peak_kb` is the job's `mem_peak`: non-positive when the executor had no process +/// to sample or the reading failed, and those are not observations. +pub(crate) fn record_job_memory_peak(tag: &str, mem_peak_kb: i32) { + if mem_peak_kb <= 0 { + return; + } + let bytes = mem_peak_kb as u64 * 1024; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.job_memory_peak + .with_label_values(&[tag]) + .observe(bytes as f64); + } + otel_record_job_memory_peak(tag, bytes); +} diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 0490ed1754..e5b84f39b9 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -21,6 +21,7 @@ use windmill_common::{ error::{self, Error}, flow_status::FlowJobDuration, jobs::JobKind, + runnable_job_stats::{accumulate_runnable_job_stats, flush_runnable_job_stats}, utils::WarnAfterExt, worker::{error_to_value, to_raw_value, Connection, WORKER_GROUP}, worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap}, @@ -32,8 +33,8 @@ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, - parse_result_object, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, - ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, + parse_result_object, CanceledBy, FlowRunners, JobCompleted, JobResourceUsage, MiniCompletedJob, + MiniPulledJob, ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -231,6 +232,10 @@ async fn process_jc( let duration_ms = jc.duration.clone(); let script_lang = jc.job.script_lang.clone(); let workspace_id = jc.job.workspace_id.clone(); + let job_kind = jc.job.kind; + let runnable_path = jc.job.runnable_path.clone(); + let mem_peak = jc.mem_peak; + let resource_usage = jc.resource_usage.take(); let root_job = handle_receive_completed_job( jc, @@ -269,6 +274,17 @@ async fn process_jc( duration_ms, ) .await; + accumulate_runnable_job_stats( + &workspace_id, + job_kind, + runnable_path.as_deref(), + resource_usage + .as_ref() + .map_or(WORKER_GROUP.as_str(), |u| u.worker_group.as_str()), + duration_ms, + mem_peak, + resource_usage.as_ref().and_then(|u| u.cpu_time_ms), + ); } success @@ -317,6 +333,14 @@ pub fn start_background_processor( let flush_handle = tokio::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(900)); // Flush every 15 min + // Once per hourly bucket, at an offset of its own per worker so that a fleet + // started together does not write the same rows in the same second. + let runnable_period = tokio::time::Duration::from_secs(3600); + let mut runnable_interval = tokio::time::interval_at( + tokio::time::Instant::now() + runnable_period.mul_f64(rand::random::()), + runnable_period, + ); + loop { tokio::select! { _ = interval.tick() => { @@ -324,6 +348,11 @@ pub fn start_background_processor( tracing::error!("Failed to flush worker group job stats: {}", e); } } + _ = runnable_interval.tick() => { + if let Err(e) = flush_runnable_job_stats(&db_clone).await { + tracing::error!("Failed to flush runnable job stats: {}", e); + } + } _ = killpill_rx_clone.recv() => { tracing::info!("bg processor received killpill signal, flushing remaining stats"); break; @@ -524,6 +553,18 @@ pub fn start_background_processor( Ok(Err(join_err)) => tracing::error!("Stats flush task failed: {}", join_err), Err(_) => tracing::error!("Stats flush timed out after 10 seconds"), } + // The periodic task stops at the killpill, before the jobs drained above were + // accumulated: without this a short-lived worker would lose its whole rollup. + match tokio::time::timeout( + std::time::Duration::from_secs(10), + flush_runnable_job_stats(&db), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => tracing::error!("Failed to flush runnable job stats: {}", e), + Err(_) => tracing::error!("Runnable job stats flush timed out after 10 seconds"), + } job_completed_processor_is_done.store(true, Ordering::SeqCst); @@ -555,6 +596,7 @@ pub async fn process_result( job_dir: &str, job_completed_tx: JobCompletedSender, mem_peak: i32, + cpu_time_ms: Option, canceled_by: Option, cached_res_path: Option, token: &str, @@ -565,6 +607,7 @@ pub async fn process_result( has_stream: bool, flow_runners: Option>, ) -> error::Result { + let resource_usage = Some(JobResourceUsage { worker_group: WORKER_GROUP.clone(), cpu_time_ms }); match result { Ok(result) => { send_job_completed( @@ -575,6 +618,7 @@ pub async fn process_result( result, result_columns, mem_peak, + resource_usage, canceled_by, success: true, cached_res_path, @@ -658,6 +702,7 @@ pub async fn process_result( result_columns: None, preprocessed_args: None, mem_peak, + resource_usage, canceled_by, success: false, cached_res_path, @@ -1776,7 +1821,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &job, true, @@ -1835,7 +1880,10 @@ pub async fn process_completed_job( true, canceled_by, result, - started_at.map(|x| FlowJobDuration { started_at: x, duration_ms: duration }), + span.started_at.or(started_at).map(|x| FlowJobDuration { + started_at: x, + duration_ms: span.duration_ms, + }), StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, @@ -1867,9 +1915,10 @@ pub async fn process_completed_job( // `wm_failure` field) so a real runtime failure whose raw // result happens to contain a `wm_failure` field still goes // through the standard `WrappedError { error: ... }` wrap path. - let downstream_result: Arc> = if is_pre_shaped_wm_failure_result(result.get()) - { - windmill_queue::add_completed_job_pre_shaped_failure( + let (downstream_result, span): (Arc>, _) = if is_pre_shaped_wm_failure_result( + result.get(), + ) { + let span = windmill_queue::add_completed_job_pre_shaped_failure( db, &job, mem_peak.to_owned(), @@ -1880,9 +1929,9 @@ pub async fn process_completed_job( None, ) .await?; - result.clone() + (result.clone(), span) } else { - let wrapped = add_completed_job_error( + let (wrapped, span) = windmill_queue::add_completed_job_error_with_span( db, &job, mem_peak.to_owned(), @@ -1895,7 +1944,10 @@ pub async fn process_completed_job( None, ) .await?; - Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()) + ( + Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()), + span, + ) }; #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { @@ -1927,12 +1979,19 @@ pub async fn process_completed_job( false, canceled_by, downstream_result, - duration.and_then(|d| { - job.started_at.map(|started_at| FlowJobDuration { - started_at: started_at, - duration_ms: d, + if span.outlasts_run(job.started_at) { + span.started_at.map(|started_at| FlowJobDuration { + started_at, + duration_ms: span.duration_ms, }) - }), + } else { + duration.and_then(|d| { + job.started_at.map(|started_at| FlowJobDuration { + started_at: started_at, + duration_ms: d, + }) + }) + }, StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 24b07eacb9..411dd520b9 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -873,7 +873,8 @@ fn convert_typ_val(arg_t: String, arg_v: Value) -> Value { let mut v = arg_v; if !v.is_string() { - // if not string, convert to string for api request + // The SQL API has no ARRAY/VARIANT binding type: a list or object + // is bound as its JSON text, which the query reads with PARSE_JSON(?). v = json!(v.to_string()); } @@ -935,3 +936,16 @@ fn parse_val(value: &Value, typ: &str) -> Value { )) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn list_arg_binds_as_json_text() { + assert_eq!( + convert_typ_val("varchar[]".to_string(), json!(["sales", "g/ops"])), + json!({ "type": "TEXT", "value": r#"["sales","g/ops"]"# }) + ); + } +} diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 45d76c74e8..3cc9c1e23a 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -113,6 +113,11 @@ pub enum WacPark { /// handler — falls back to `now() - started_at`. Left pointing at the first segment, /// that fallback reports the whole sleep or approval wait as execution time. /// +/// The first park keeps that first start in `extras.wac_first_started_at`, so the +/// completed row can still say when the workflow began and how long it took overall +/// (`Completion::execute`). The queue row's `started_at` stays per segment. The inline +/// checkpoint requeue clears `started_at` on its own and keeps the first start the same way. +/// /// Call it before any write to the parent's `v2_job_status` row in the same /// transaction: a child's completion locks the queue row and then the status row /// (`record_child_completion`), and taking them the other way round here can @@ -156,7 +161,10 @@ pub async fn suspend_wac_parent( sqlx::query!( "UPDATE v2_job_queue - SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null + SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null, + extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras + ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at)) + END WHERE id = $1 AND workspace_id = $2", job_id, w_id, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 80ca37acb8..3aa377c904 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -155,7 +155,7 @@ use crate::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, tally_unfinished_dependency_deploy, }, - worker_utils::{insert_ping, queue_vacuum, update_worker_ping_full}, + worker_utils::{insert_ping, mark_worker_draining, queue_vacuum, update_worker_ping_full}, }; #[cfg(feature = "rust")] @@ -215,6 +215,8 @@ lazy_static::lazy_static! { pub static ref PY311_CACHE_DIR: String = format!("{}python_3_11", *ROOT_CACHE_DIR); pub static ref PY312_CACHE_DIR: String = format!("{}python_3_12", *ROOT_CACHE_DIR); pub static ref PY313_CACHE_DIR: String = format!("{}python_3_13", *ROOT_CACHE_DIR); + pub static ref PY314_CACHE_DIR: String = format!("{}python_3_14", *ROOT_CACHE_DIR); + pub static ref PY315_CACHE_DIR: String = format!("{}python_3_15", *ROOT_CACHE_DIR); pub static ref TAR_JAVA_CACHE_DIR: String = format!("{}tar/java", *ROOT_CACHE_DIR); @@ -949,6 +951,22 @@ pub async fn workspace_registry_cache_suffix(w_id: &str) -> String { } } +/// The registry overrides of a workspace, empty when it has none. +/// +/// One snapshot: registry settings reload under running jobs, so a caller whose cache key +/// has to agree with the registry values it used reads both from here rather than calling +/// [`workspace_registry_cache_suffix`] and a `read_ee_registry_*` helper separately. +pub async fn workspace_registry_overrides( + w_id: &str, +) -> std::collections::HashMap { + let registries = WORKSPACE_REGISTRIES.read().await; + registries + .as_ref() + .and_then(|m| m.get(w_id)) + .cloned() + .unwrap_or_default() +} + /// The name a build artifact is cached under, derived from `base` — the runnable's own /// cache-key input — and its inline modules. /// @@ -2005,6 +2023,7 @@ pub fn create_span_with_name( root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, + worker_group = %*WORKER_GROUP, hostname = field::Empty, tag = %arc_job.tag, language = field::Empty, @@ -2156,6 +2175,7 @@ pub fn log_context_for_job( job_id: Some(arc_job.id.to_string()), workspace_id: Some(arc_job.workspace_id.clone()), worker: Some(worker_name.to_string()), + worker_group: Some(WORKER_GROUP.clone()), tag: Some(arc_job.tag.clone()), job_kind: Some(arc_job.kind.as_str().to_string()), created_by: Some(arc_job.created_by.clone()), @@ -2229,6 +2249,7 @@ pub async fn handle_all_job_kind_error( ))), result_columns: None, mem_peak: 0, + resource_usage: None, canceled_by: None, success: false, cached_res_path: None, @@ -2695,7 +2716,7 @@ pub async fn run_worker( let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1); let mut reported_ip = cached_ip(); - let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, conn) + let previous_jobs_executed = insert_ping(hostname, &worker_name, reported_ip, true, conn) .await .expect("initial ping could be sent"); @@ -3142,6 +3163,31 @@ pub async fn run_worker( let mut killpill_rx2 = killpill_rx.resubscribe(); + // The loop below only sees the killpill once the running job is done, which is too late + // for that job to learn its worker is going away. + { + let mut killpill_rx = killpill_rx.resubscribe(); + let conn = conn.clone(); + let worker_name = worker_name.clone(); + tokio::spawn(async move { + if matches!( + killpill_rx.recv().await, + Err(broadcast::error::RecvError::Closed) + ) { + return; + } + for attempt in 1..=3 { + match mark_worker_draining(&conn, &worker_name).await { + Ok(()) => return, + Err(e) => { + tracing::warn!(worker = %worker_name, "failed to mark worker as draining (attempt {attempt}/3): {e:#}"); + tokio::time::sleep(Duration::from_secs(2)).await; + } + } + } + }); + } + loop { let last_processing_duration_secs = last_processing_duration.load(Ordering::SeqCst); if last_processing_duration_secs > 5 { @@ -3217,7 +3263,7 @@ pub async fn run_worker( // which costs at most the last job's id here: no job of this worker is in flight at this // point in the loop, and the next one refills them. if ip_just_resolved && conn.as_sql().is_none() { - if let Err(e) = insert_ping(hostname, &worker_name, ip, &conn).await { + if let Err(e) = insert_ping(hostname, &worker_name, ip, false, &conn).await { tracing::warn!( worker = %worker_name, hostname = %hostname, "failed to re-register with the resolved external IP: {e}" @@ -3406,6 +3452,10 @@ pub async fn run_worker( tokio::time::sleep(Duration::from_millis(200)).await; continue; } + } else if WORKER_CONFIG.load().paused { + // Checked after the same-worker channel: a flow this worker already runs + // still needs it for its remaining same-worker steps. + Ok(None) } else { match &conn { Connection::Sql(db) => { @@ -3725,6 +3775,7 @@ pub async fn run_worker( result: Arc::new(empty_result()), result_columns: None, mem_peak: 0, + resource_usage: None, cached_res_path: None, token: "".to_string(), canceled_by: None, @@ -4716,6 +4767,7 @@ pub async fn handle_queued_job( result, result_columns: None, mem_peak: 0, + resource_usage: None, canceled_by: None, success: true, cached_res_path: None, @@ -5065,6 +5117,8 @@ pub async fn handle_queued_job( let cjob = MiniCompletedJob::from(job.to_owned()); drop(job); + // Taken ahead of the returns below so that none of them leaves the entry behind. + let cpu_time_ms = crate::handle_child::take_job_cpu_time_ms(&cjob.id); //it's a test job, no need to update the db if cjob.workspace_id == "" { return Ok(JobOutcome::Completed); @@ -5081,14 +5135,27 @@ pub async fn handle_queued_job( .is_err_and(|err| matches!(err, &Error::WacSuspended(_))) { // WAC v2 job suspended while waiting for child jobs — don't complete it + // Only a worker with a database connection flushes the rollup. + if let Connection::Sql(_) = conn { + windmill_common::runnable_job_stats::accumulate_runnable_round( + &cjob.workspace_id, + cjob.kind, + cjob.runnable_path.as_deref(), + &WORKER_GROUP, + started.elapsed().as_millis() as i64, + cpu_time_ms, + ); + } return Ok(JobOutcome::Completed); } + crate::resource_metrics::record_job_memory_peak(&cjob.tag, mem_peak); process_result( cjob, result.map(|x| Arc::new(x)), job_dir, job_completed_tx, mem_peak, + cpu_time_ms, canceled_by, cached_res_path, &client.token, @@ -7186,7 +7253,7 @@ pub fn init_worker_internal_server_inline_utils( base_internal_url, killpill_rx: Arc::new(killpill_rx), run_inline_preview_script: Arc::new(|params| { - let job = MiniPulledJob::new_inline( + let mut job = MiniPulledJob::new_inline( params.workspace_id, params.args, params.created_by, @@ -7198,6 +7265,7 @@ pub fn init_worker_internal_server_inline_utils( "inline_preview".to_string(), Some(params.lang), ); + job.permissioned_as_end_user_email = params.end_user_email; Box::pin(async move { let mut mem_peak: i32 = -1; let mut canceled_by: Option = None; @@ -7247,7 +7315,7 @@ pub fn init_worker_internal_server_inline_utils( let content_info = get_script_content_by_hash(&script_hash, ¶ms.workspace_id, ¶ms.conn) .await?; - let job = MiniPulledJob::new_inline( + let mut job = MiniPulledJob::new_inline( params.workspace_id, params.args, params.created_by, @@ -7259,6 +7327,7 @@ pub fn init_worker_internal_server_inline_utils( "inline_run".to_string(), content_info.language, ); + job.permissioned_as_end_user_email = params.end_user_email; let mut mem_peak: i32 = -1; let mut canceled_by: Option = None; let mut column_order: Option> = None; diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 6d76d0d771..4f67780f18 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2097,7 +2097,7 @@ pub async fn update_flow_status_after_job_completion_internal( ) .await?; let duration = if success { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, true, @@ -2111,9 +2111,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms } else { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, false, @@ -2131,7 +2131,7 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms }; flow_job_duration = flow_job .started_at diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index b608601991..6189d39b8d 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1244,6 +1244,7 @@ pub async fn handle_flow_dependency_job( &job.workspace_id, flow.failure_module.as_ref(), flow.same_worker, + flow.preserve_step_tags, ) .await?; @@ -2134,6 +2135,7 @@ async fn insert_flow_modules<'c>( workspace_id: &str, failure_module: Option<&Box>, same_worker: bool, + preserve_step_tags: bool, modules: &mut Vec, modules_node: &mut Option, ) -> Result> { @@ -2144,6 +2146,7 @@ async fn insert_flow_modules<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, )) .await?; if modules.is_empty() || crate::worker_flow::is_simple_modules(modules, failure_module) { @@ -2160,6 +2163,7 @@ async fn insert_flow_modules<'c>( modules: std::mem::take(modules), failure_module: failure_module.cloned(), same_worker, + preserve_step_tags, ..Default::default() }))), None, @@ -2176,6 +2180,7 @@ async fn reduce_flow<'c>( workspace_id: &str, failure_module: Option<&Box>, same_worker: bool, + preserve_step_tags: bool, ) -> Result> { use FlowModuleValue::*; for module in &mut *modules { @@ -2234,6 +2239,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, modules, modules_node, ) @@ -2247,6 +2253,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, &mut branch.modules, &mut branch.modules_node, ) @@ -2258,6 +2265,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, default, default_node, ) @@ -2271,6 +2279,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, &mut branch.modules, &mut branch.modules_node, ) diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 3a313e9360..c634c13b6a 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -7,9 +7,9 @@ use windmill_common::{ external_ip::UNKNOWN_IP, worker::{ get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, - insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query, - update_worker_ping_main_loop_query, Connection, Ping, PingType, NATIVE_MODE_RESOLVED, - WORKER_CONFIG, WORKER_GROUP, + insert_ping_query, set_worker_draining_query, update_job_ping_query, + update_worker_ping_from_job_query, update_worker_ping_main_loop_query, Connection, Ping, + PingType, NATIVE_MODE_RESOLVED, WORKER_CONFIG, WORKER_GROUP, }, KillpillSender, DB, }; @@ -39,17 +39,26 @@ pub(crate) async fn update_worker_ping_full( let wm_memory_usage = get_windmill_memory_usage(); let (vcpus, memory) = if read_cgroups { - (get_vcpus(), get_memory()) + let memory = get_memory(); + crate::resource_metrics::set_memory_limit(memory); + (get_vcpus(), memory) } else { (None, None) }; + let occupancy = occupancy_metrics.update_occupancy_metrics(); + crate::resource_metrics::record_worker_resources( + worker_name, + memory_usage, + wm_memory_usage, + Some(&occupancy), + ); let OccupancyResult { occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, - } = occupancy_metrics.update_occupancy_metrics(); + } = occupancy; let ping_start = std::time::Instant::now(); if let Err(e) = (|| { @@ -164,6 +173,7 @@ async fn update_worker_ping_full_inner( wm_memory_usage: get_windmill_memory_usage(), job_isolation: None, native_mode: Some(native_mode), + draining: None, ping_type: PingType::MainLoop, }, ) @@ -179,6 +189,7 @@ pub async fn insert_ping( worker_instance: &str, worker_name: &str, ip: Option<&str>, + reset_draining: bool, db: &Connection, ) -> anyhow::Result { let (tags, dw, dws, native_mode) = { @@ -224,6 +235,7 @@ pub async fn insert_ping( memory, job_isolation, native_mode, + reset_draining.then_some(false), db, ) .await; @@ -256,6 +268,7 @@ pub async fn insert_ping( wm_memory_usage: get_windmill_memory_usage(), job_isolation, native_mode: Some(native_mode), + draining: reset_draining.then_some(false), ping_type: PingType::Initial, }, ) @@ -314,6 +327,7 @@ pub async fn update_worker_ping_from_job( &Ping { last_job_executed: Some(job_id.clone()), last_job_workspace_id: Some(w_id.to_string()), + draining: None, ping_type: PingType::Job, worker_instance: None, ip: None, @@ -342,6 +356,45 @@ pub async fn update_worker_ping_from_job( Ok(()) } +/// Publishes on the worker's ping row that it has received its killpill, so the job it is +/// still running can find out and exit early. +pub async fn mark_worker_draining(conn: &Connection, worker_name: &str) -> anyhow::Result<()> { + match conn { + Connection::Sql(db) => set_worker_draining_query(worker_name, db).await, + Connection::Http(client) => { + client + .post::( + UPDATE_PING_URL, + None, + &Ping { + last_job_executed: None, + last_job_workspace_id: None, + draining: None, + ping_type: PingType::Draining, + worker_instance: None, + ip: None, + tags: None, + dw: None, + dws: None, + version: None, + vcpus: None, + memory: None, + memory_usage: None, + wm_memory_usage: None, + jobs_executed: None, + occupancy_rate: None, + occupancy_rate_15s: None, + occupancy_rate_5m: None, + occupancy_rate_30m: None, + job_isolation: None, + native_mode: None, + }, + ) + .await + } + } +} + pub async fn ping_job_status( conn: &Connection, job_id: &Uuid, diff --git a/benchmarks/README.md b/benchmarks/README.md index fc758c006f..1dc4818848 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -22,12 +22,12 @@ deno run -A benchmark_suite.ts -c suite_wac.json ### Script benchmarks - `noop` — Empty jobs (measures pure scheduling overhead) -- `deno`, `bun`, `python`, `go`, `bash` — Language runtimes +- `bun`, `python`, `go`, `bash` — Language runtimes - `nativets` — BunNative (no isolation) - `dedicated`, `dedicated_nativets` — Dedicated worker mode ### Flow benchmarks -- `2steps` — 2-step flow (deno + identity) +- `2steps` — 2-step flow (bun + identity) - `bigscriptinflow` — Flow with large raw bash script - `flow_seq_2_bun` — 2 sequential bun steps - `flow_par_2_bun` — 2 parallel bun steps (branchall) diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 5f075d2034..5aa75b5864 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -37,7 +37,7 @@ async function verifyOutputs(uuids: string[], workspace: string) { console.log(`Incorrect results: ${incorrectResults}`); } -export const NON_TEST_TAGS = ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"] +export const NON_TEST_TAGS = ["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"] const FLOW_COMPARISON_KINDS = ["flow_seq_2_bun", "flow_par_2_bun", "flow_seq_3_bun"]; export async function main({ @@ -148,7 +148,7 @@ export async function main({ } if ( - ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( + ["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( kind ) ) { @@ -169,7 +169,7 @@ export async function main({ kind: "noop", }); } else if ( - ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( + ["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( kind ) ) { @@ -414,7 +414,7 @@ if (import.meta.main) { ) .option( "--kind ", - "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun", + "Specifiy the benchmark kind among: identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun", { required: true, } diff --git a/benchmarks/graphs_config.json b/benchmarks/graphs_config.json index 8e37695d66..61b5bdddc8 100644 --- a/benchmarks/graphs_config.json +++ b/benchmarks/graphs_config.json @@ -49,26 +49,6 @@ } ] }, - { - "graph_title": "deno throughput benchmark", - "benchmarks": [ - { - "kind": "deno", - "workers": 1, - "label": "1 worker" - }, - { - "kind": "deno", - "workers": 4, - "label": "4 workers" - }, - { - "kind": "deno", - "workers": 8, - "label": "8 workers" - } - ] - }, { "graph_title": "bun throughput benchmark", "benchmarks": [ @@ -170,7 +150,7 @@ ] }, { - "graph_title": "go vs python vs deno vs bun vs bash vs nativets", + "graph_title": "go vs python vs bun vs bash vs nativets", "benchmarks": [ { "kind": "go", @@ -182,11 +162,6 @@ "workers": 1, "label": "python" }, - { - "kind": "deno", - "workers": 1, - "label": "deno" - }, { "kind": "bun", "workers": 1, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2b5215da99..34e4da5cf3 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.826.0"; +export const VERSION = "v1.830.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ @@ -87,10 +87,6 @@ export async function createBenchScript( schemaProperties = { uuid: { default: null, description: "", type: "string" }, }; - } else if (scriptPattern === "deno") { - scriptContent = - 'export function main(){ return Deno.env.get("WM_JOB_ID"); }'; - language = "deno"; } else if (scriptPattern === "nativets") { scriptContent = '//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }'; @@ -256,10 +252,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { id: "a", value: { input_transforms: {}, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: - 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + 'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }', }, }, { @@ -277,7 +273,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { expr: "results.a", }, }, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: "export function main(x: string){ return x; }", }, @@ -298,10 +294,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { id: "a", value: { input_transforms: {}, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: - 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + 'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }', }, }, { @@ -321,7 +317,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { expr: "results.a", }, }, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: "export function main(x: string){ return x; }", }, @@ -339,7 +335,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { expr: "results.a", }, }, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: "export function main(x: string){ return x; }", }, @@ -490,10 +486,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { id: "a", value: { input_transforms: {}, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: - 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + 'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }', }, }, { diff --git a/benchmarks/main.ts b/benchmarks/main.ts index 1f681a9c03..bf2f998854 100644 --- a/benchmarks/main.ts +++ b/benchmarks/main.ts @@ -175,11 +175,11 @@ export async function main({ if ( !useFlows && (scriptPattern === undefined || - ["deno", "python", "go", "bash", "bun", "dedicated"].includes( + ["python", "go", "bash", "bun", "dedicated"].includes( scriptPattern )) ) { - await createBenchScript(scriptPattern || "deno", workspace); + await createBenchScript(scriptPattern || "bun", workspace); } let workers: Worker[] = new Array(num_workers); @@ -458,7 +458,7 @@ if (import.meta.main) { ) .option( "--script-pattern ", - "Use a different script pattern among: deno, identity, python, go, bash, dedicated, bun (Default deno)" + "Use a different script pattern among: bun, identity, python, go, bash, dedicated (Default bun)" ) .option("--custom ", "Use custom actions during bench") .option( diff --git a/benchmarks/suite_config.json b/benchmarks/suite_config.json index a2704a46fe..ed50c60b1e 100644 --- a/benchmarks/suite_config.json +++ b/benchmarks/suite_config.json @@ -7,10 +7,6 @@ "kind": "2steps", "jobs": 250 }, - { - "kind": "deno", - "jobs": 500 - }, { "kind": "bun", "jobs": 500 diff --git a/benchmarks/worker.ts b/benchmarks/worker.ts index ae06fa9779..ce2d695771 100644 --- a/benchmarks/worker.ts +++ b/benchmarks/worker.ts @@ -118,7 +118,7 @@ while (cont) { } else { uuid = await windmill.JobService.runScriptByPath({ workspace: config.workspace_id, - path: "f/benchmarks/" + (config.scriptPattern || "deno"), + path: "f/benchmarks/" + (config.scriptPattern || "bun"), requestBody: {}, }); } diff --git a/chat-sdk/README.md b/chat-sdk/README.md index 62a5467a47..006d5407d8 100644 --- a/chat-sdk/README.md +++ b/chat-sdk/README.md @@ -1,21 +1,21 @@ -# windmill-chat +# @windmill-labs/chat-sdk Build a chat interface on a Windmill flow deployed in **chat mode**, from any frontend or from a Windmill raw app. The library is headless: it runs the flow, follows the answer as it streams, keeps the conversation history, and hands you state to render. ``` -npm install windmill-chat +npm install @windmill-labs/chat-sdk ``` -No runtime dependencies. Optional peers: `react` for `windmill-chat/react`, `ai` for -`windmill-chat/ai-sdk`, `@assistant-ui/react` for `windmill-chat/assistant-ui`. +No runtime dependencies. Optional peers: `react` for `@windmill-labs/chat-sdk/react`, `ai` for +`@windmill-labs/chat-sdk/ai-sdk`, `@assistant-ui/react` for `@windmill-labs/chat-sdk/assistant-ui`. | You build the UI with | Import | You get | |---|---|---| -| Vercel AI SDK `useChat`, AI Elements | `windmill-chat/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | -| assistant-ui | `windmill-chat/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | -| Your own components | `windmill-chat/react` or `windmill-chat` | a hook / a store with messages, status and actions | +| Vercel AI SDK `useChat`, AI Elements | `@windmill-labs/chat-sdk/ai-sdk` | a `ChatTransport`: `useChat({ transport })`, nothing else changes | +| assistant-ui | `@windmill-labs/chat-sdk/assistant-ui` | a runtime for `AssistantRuntimeProvider`, threads included | +| Your own components | `@windmill-labs/chat-sdk/react` or `@windmill-labs/chat-sdk` | a hook / a store with messages, status and actions | ## The flow @@ -31,7 +31,7 @@ so an AI agent step remembers earlier turns. The answer is: ```tsx import { useChat } from '@ai-sdk/react' -import { createWindmillChatTransport } from 'windmill-chat/ai-sdk' +import { createWindmillChatTransport } from '@windmill-labs/chat-sdk/ai-sdk' const transport = createWindmillChatTransport({ baseUrl: 'https://app.windmill.dev', @@ -65,7 +65,7 @@ explanatory error. ```tsx import { AssistantRuntimeProvider } from '@assistant-ui/react' -import { useWindmillRuntime } from 'windmill-chat/assistant-ui' +import { useWindmillRuntime } from '@windmill-labs/chat-sdk/assistant-ui' export function Support() { const runtime = useWindmillRuntime({ baseUrl, workspace, flowPath, token }) @@ -84,7 +84,7 @@ through `Reasoning`. It takes the same options as `useWindmillChat` below. ## React ```tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' export function Support() { const chat = useWindmillChat({ @@ -155,7 +155,7 @@ change, and returns the unsubscribe function. That is the Svelte store contract, `$chat` works as is; other frameworks wrap it in a few lines. ```ts -import { createChat } from 'windmill-chat' +import { createChat } from '@windmill-labs/chat-sdk' const chat = createChat({ baseUrl, workspace, flowPath, token }) chat.subscribe((state) => render(state)) @@ -164,7 +164,7 @@ await chat.sendMessage('Hello') ```svelte @@ -331,8 +331,8 @@ custom integrations. ## For AI coding agents When asked to add a chat over a Windmill flow: the flow must be deployed with chat mode -on. Pick the entry point from the table at the top (`useChat` → `windmill-chat/ai-sdk`, -assistant-ui → `windmill-chat/assistant-ui`, otherwise `windmill-chat/react`). Inside a +on. Pick the entry point from the table at the top (`useChat` → `@windmill-labs/chat-sdk/ai-sdk`, +assistant-ui → `@windmill-labs/chat-sdk/assistant-ui`, otherwise `@windmill-labs/chat-sdk/react`). Inside a Windmill raw app pass only `flowPath`. Elsewhere pass `baseUrl`, `workspace` and a `token`; for a public page use a token scoped to `jobs:run:flows:` and leave `history` at its default. Render `role`, `content`, `pending`, `success` and diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index fcc48449cf..74f6914f37 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { - "name": "windmill-chat", - "version": "1.826.0", + "name": "@windmill-labs/chat-sdk", + "version": "1.830.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "windmill-chat", - "version": "1.826.0", + "name": "@windmill-labs/chat-sdk", + "version": "1.830.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index 33b6c43fa3..26f5f785be 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { - "name": "windmill-chat", + "name": "@windmill-labs/chat-sdk", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.826.0", + "version": "1.830.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/bun.lock b/cli/bun.lock index 0e0885f563..db6fd253bf 100644 --- a/cli/bun.lock +++ b/cli/bun.lock @@ -19,14 +19,14 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.56.8", "tar-stream": "^3.1.7", - "windmill-parser-wasm-asset": "1.753.0", + "windmill-parser-wasm-asset": "1.830.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", - "windmill-parser-wasm-py": "1.782.0", - "windmill-parser-wasm-py-imports": "1.693.1", + "windmill-parser-wasm-py": "1.830.0", + "windmill-parser-wasm-py-imports": "1.830.0", "windmill-parser-wasm-r": "1.668.1", "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", @@ -291,7 +291,7 @@ "utility-types": ["utility-types@3.11.0", "", {}, "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw=="], - "windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.753.0", "", {}, "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg=="], + "windmill-parser-wasm-asset": ["windmill-parser-wasm-asset@1.830.0", "", {}, "sha512-q4HSaMOd5cF8pmfPcEGiGeQbO8vhtv89us7zbO12IlgpxpWT8ojvw66J75y4yUt1PaEuaoztSaX8Gkxxn/6MXw=="], "windmill-parser-wasm-csharp": ["windmill-parser-wasm-csharp@1.510.1", "", {}, "sha512-qm09YmnbeYHLwYn1jUnObVzPhYO9NZKMlIO7nlo7zPJBXqksgG5fK/KCtwGw9rChrnz+DsvM9wP5FhrwRLMtwQ=="], @@ -303,9 +303,9 @@ "windmill-parser-wasm-php": ["windmill-parser-wasm-php@1.647.1", "", {}, "sha512-u2qaMkupSdhJibxvkLh3r/y36IARvnYNTLXWvOKxcQ0G/BPUB4+yF5o/yf47vv9zUV5WZv4mrdsKDt/pZDYeDg=="], - "windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.782.0", "", {}, "sha512-ZnpEvhbffaJI+Uq7HH1mVO6kUgR4uiAIIxaLa30Pqb1/oz0h4abT6W7bFOApIE5X9SDOuycoaDBEr9Ogm6ZLzQ=="], + "windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.830.0", "", {}, "sha512-+P4mIyp3lXkgjsr3/20Oc31ank8orQDd4K/v8TnjTQnCyxyy+VB9rynLESnkH6OlcEVSk2qf9pDM9c367E8DoQ=="], - "windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.693.1", "", {}, "sha512-FC0KbREe2G/sa/9kYIR930wmWw+VL6PvEIqg12J3dsJes3A+0x5JIUPT/jeD+c24DrG0ko/Ub7yDnYs56Bem7g=="], + "windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.830.0", "", {}, "sha512-Pi9gHStNFTEqfcDrjSxK5SnI/1dTG4xKcVUD3BHlR1aQ7E/VxkTD/D0INJm7mNC2usKb2bO21NMdKmGh+RAXEw=="], "windmill-parser-wasm-r": ["windmill-parser-wasm-r@1.668.1", "", {}, "sha512-5YNeUibxpNBvYrxCgQcz1PxGhTFx2CyEpg2udtIhq7bx0d4gF/KDZVupMeQmAObmrEtTSFGUWNRJ4zXSWNrSpQ=="], diff --git a/cli/package.json b/cli/package.json index 5e7623a7ab..5efe8a2499 100644 --- a/cli/package.json +++ b/cli/package.json @@ -31,14 +31,14 @@ "pg-gateway": "0.3.0-beta.4", "svelte": "^5.56.8", "tar-stream": "^3.1.7", - "windmill-parser-wasm-asset": "1.753.0", + "windmill-parser-wasm-asset": "1.830.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", - "windmill-parser-wasm-py": "1.782.0", - "windmill-parser-wasm-py-imports": "1.693.1", + "windmill-parser-wasm-py": "1.830.0", + "windmill-parser-wasm-py-imports": "1.830.0", "windmill-parser-wasm-r": "1.668.1", "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", diff --git a/cli/src/commands/app/app.ts b/cli/src/commands/app/app.ts index 0d4ad634c8..0060c2c422 100644 --- a/cli/src/commands/app/app.ts +++ b/cli/src/commands/app/app.ts @@ -32,6 +32,8 @@ export interface AppFile { guests?: boolean; value: any; public?: boolean; + sandbox?: boolean; + frontend_api?: string[]; summary: string; policy: Policy; // Mirrors granular ACLs on the app path. Omitted from app.yaml when no @@ -113,6 +115,14 @@ export function replaceInlineScripts( } return []; } +/** `parent_version` is the editor's stale-draft base: a version id that changes on + * every deploy and means nothing in another workspace, so it is kept out of the + * tracked file and out of what a push deploys. */ +export function stripEditorState(app: any) { + if (app?.value && typeof app.value == "object") { + delete app.value.parent_version; + } +} export function isExecutionModeAnonymous(app: any) { return app?.["policy"]?.["execution_mode"] == "anonymous"; } @@ -160,6 +170,100 @@ export function executionModeForPush( } return deployedPolicy?.execution_mode === "viewer" ? "viewer" : "publisher"; } +/** Mirrors `FRONTEND_SDK_ALLOWED_SCOPES` in the backend's `apps.rs`, which stays + * the authority: this copy only fails a push before it bundles. */ +export const FRONTEND_API_SCOPES = [ + "jobs:run", + "jobs:read", + "users:read", + "resources:read", + "variables:read", + "flow_conversations:read", + "flow_conversations:write", +]; + +export type AppIsolation = { sandbox: boolean; frontendApi: string[] }; + +/** Write `sandbox` and `frontend_api` on a pulled app, only when they are on, so + * an app using neither keeps the file it had. Scopes of an unsandboxed app are + * inert and stay out: a file stating them without `sandbox` does not push. */ +export function markIsolationFromPolicy(app: any, rawApp: boolean) { + if (app?.["policy"]?.["sandbox"] !== true) { + return; + } + app.sandbox = true; + const scopes = app.policy.frontend_sdk_scopes; + if (rawApp && Array.isArray(scopes) && scopes.length > 0) { + app.frontend_api = scopes; + } +} + +/** The isolation the tracked file states, or `undefined` when it has no `sandbox` + * key and the deployed policy carries over. A file that has the key states both + * fields: `sandbox: false` turns isolation off, and a missing `frontend_api` + * means no scopes. */ +export function statedIsolation( + localApp: any, + rawApp: boolean, +): AppIsolation | undefined { + const sandbox = localApp?.["sandbox"]; + const scopes = localApp?.["frontend_api"] ?? []; + if (sandbox !== undefined && typeof sandbox !== "boolean") { + throw new Error("'sandbox' must be true or false"); + } + if ( + !Array.isArray(scopes) || + scopes.some((s: unknown) => typeof s !== "string") + ) { + throw new Error("'frontend_api' must be a list of scopes"); + } + if (scopes.length > 0 && !rawApp) { + throw new Error("'frontend_api' is only available to full-code apps"); + } + const seen = new Set(); + for (const scope of scopes) { + if (!FRONTEND_API_SCOPES.includes(scope)) { + throw new Error( + `Invalid frontend_api scope '${scope}'. Allowed scopes: ${FRONTEND_API_SCOPES.join(", ")}`, + ); + } + if (seen.has(scope)) { + throw new Error(`Duplicate frontend_api scope '${scope}'`); + } + seen.add(scope); + } + if (scopes.length > 0 && sandbox !== true) { + throw new Error( + "'frontend_api' requires 'sandbox: true': an unsandboxed app gets no frontend API token", + ); + } + return sandbox === undefined ? undefined : { sandbox, frontendApi: scopes }; +} + +/** Whether the deployed policy already has the stated isolation. The file's two + * keys are kept out of the superset comparison against the deployed app, which + * could not see a key the file dropped. */ +export function isolationUpToDate( + isolation: AppIsolation | undefined, + deployedPolicy: Policy | undefined, + rawApp: boolean, +): boolean { + if (!isolation) { + return true; + } + if ((deployedPolicy?.sandbox === true) !== isolation.sandbox) { + return false; + } + if (!rawApp) { + return true; + } + const deployed = deployedPolicy?.frontend_sdk_scopes ?? []; + return ( + deployed.length === isolation.frontendApi.length && + deployed.every((s, i) => s === isolation.frontendApi[i]) + ); +} + export async function pushApp( workspace: string, remotePath: string, @@ -198,6 +302,8 @@ export async function pushApp( } const path = localPath + "app.yaml"; const localApp = (await yamlParseFile(path)) as AppFile; + stripEditorState(localApp); + const isolation = statedIsolation(localApp, false); replaceInlineScripts(localApp.value, localPath, true); // On create the backend applies folder defaults, so there is nothing to preserve. @@ -210,17 +316,26 @@ export async function pushApp( localApp, remotePath, executionModeForPush(localApp, deployedPolicy), - basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of) + basePolicy(localApp, deployedPolicy, !!preserveFields.preserve_on_behalf_of), + isolation ); // extra_perms goes through /acls/* — strip from the body so a perms-only // edit never bumps the app version (see applyExtraPermsDiff for details). - const { extra_perms: localPerms, ...localAppBody } = localApp as AppFile & { + const { + extra_perms: localPerms, + sandbox: _sandbox, + frontend_api: _frontendApi, + ...localAppBody + } = localApp as AppFile & { extra_perms?: Record; }; if (app) { - if (isSuperset(localAppBody, app)) { + if ( + isSuperset(localAppBody, app) && + isolationUpToDate(isolation, deployedPolicy, false) + ) { log.info(colors.green(`App ${remotePath} is up to date`)); } else { log.info(colors.bold.yellow(`Updating app ${remotePath}...`)); @@ -269,12 +384,13 @@ export async function generatingPolicy( app: any, path: string, executionMode: AppExecutionMode, - base: Policy | undefined + base: Policy | undefined, + isolation?: AppIsolation ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { app.policy = await windmillUtils.updatePolicy(app.value, base); - finalizeDerivedPolicy(app.policy, executionMode); + finalizeDerivedPolicy(app.policy, executionMode, isolation); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; @@ -332,10 +448,21 @@ export function preserveOnBehalfOfFields( * deployed app would keep being able to run runnables this push removed. */ export function finalizeDerivedPolicy( policy: Policy, - executionMode: AppExecutionMode + executionMode: AppExecutionMode, + isolation?: AppIsolation, + rawApp: boolean = false ) { policy.triggerables = undefined; policy.execution_mode = executionMode; + if (isolation) { + // `false` is sent rather than left out: the server reads an omitted value as + // "no opinion" and may sandbox the app anyway, hiding that the file was refused. + policy.sandbox = isolation.sandbox; + if (rawApp) { + policy.frontend_sdk_scopes = + isolation.frontendApi.length > 0 ? isolation.frontendApi : undefined; + } + } } async function list(opts: GlobalOptions & { includeDraftOnly?: boolean; json?: boolean }) { diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index 286e878cfb..287d01ebed 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -58,10 +58,10 @@ function conditionsFor(svelte: boolean): string[] { /** * Detects which frontend frameworks are present in package.json */ -export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolean } { +export function detectFrameworks(appDir: string): { svelte: boolean } { const packageJsonPath = path.join(appDir, "package.json"); if (!fs.existsSync(packageJsonPath)) { - return { svelte: false, vue: false }; + return { svelte: false }; } try { @@ -73,10 +73,9 @@ export function detectFrameworks(appDir: string): { svelte: boolean; vue: boolea return { svelte: "svelte" in allDeps, - vue: "vue" in allDeps, }; } catch { - return { svelte: false, vue: false }; + return { svelte: false }; } } @@ -256,17 +255,6 @@ export async function createFrameworkPlugins(appDir: string): Promise { plugins.push(createSveltePlugin(appDir)); } - if (frameworks.vue) { - log.info(colors.blue("🔧 Vue detected, adding vue plugin...")); - throw new Error("Vue plugin not supported yet"); - // try { - // const esbuildPluginVue = await import("esbuild-plugin-vue3"); - // plugins.push(esbuildPluginVue.default()); - // } catch (error: any) { - // log.warn(colors.yellow(`Failed to load vue plugin: ${error.message}`)); - // } - } - return plugins; } @@ -313,7 +301,7 @@ export async function createBundle( // Use the entryPoint's directory if provided, otherwise fall back to cwd. const appDir = options.entryPoint ? path.dirname(options.entryPoint) : process.cwd(); const frameworks = detectFrameworks(appDir); - const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; + const defaultEntry = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = options.entryPoint ?? defaultEntry; const outDir = options.outDir ?? "dist"; @@ -331,7 +319,7 @@ export async function createBundle( // Ensure node_modules exists in the app directory await ensureNodeModules(appDir); - // Load framework-specific plugins (svelte, vue) based on package.json + // Load framework-specific plugins (svelte) based on package.json const frameworkPlugins = await createFrameworkPlugins(appDir); // Ensure output directory exists diff --git a/cli/src/commands/app/bundle_command.ts b/cli/src/commands/app/bundle_command.ts index bef13f4268..c4119aafb0 100644 --- a/cli/src/commands/app/bundle_command.ts +++ b/cli/src/commands/app/bundle_command.ts @@ -12,10 +12,10 @@ interface BundleOptions { } /** Every entry point a raw app may have, most specific first, with the - * framework's preferred one promoted — Svelte and Vue mount from a `.ts`. */ + * framework's preferred one promoted — Svelte mounts from a `.ts`. */ function entryPointFor(appDir: string): string { const frameworks = detectFrameworks(appDir); - const candidates = frameworks.svelte || frameworks.vue + const candidates = frameworks.svelte ? ["index.ts", "index.tsx", "index.js"] : ["index.tsx", "index.ts", "index.js"]; const entry = candidates.find((c) => fs.existsSync(path.join(appDir, c))); diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index 2dac8639e2..fdf26fa7da 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -481,7 +481,7 @@ async function dev(opts: DevOptions, appFolder?: string) { // Detect frameworks to determine default entry point const frameworks = detectFrameworks(process.cwd()); - const defaultEntry = frameworks.svelte || frameworks.vue + const defaultEntry = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = opts.entry ?? defaultEntry; @@ -577,7 +577,7 @@ async function dev(opts: DevOptions, appFolder?: string) { const buildOptions = getDevBuildOptions(entryPoint, frameworks.svelte); - // Load framework-specific plugins (svelte, vue) based on package.json + // Load framework-specific plugins (svelte) based on package.json const frameworkPlugins = await createFrameworkPlugins(appDir); const wmillPlugin = { @@ -1602,7 +1602,7 @@ const command = new Command() }) .option( "--entry ", - "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)", + "Entry point file (default: index.ts for Svelte, index.tsx otherwise)", ) .option("--no-open", "Don't automatically open the browser") .option( diff --git a/cli/src/commands/app/lint.ts b/cli/src/commands/app/lint.ts index 9356b5dd3c..c90a22105d 100644 --- a/cli/src/commands/app/lint.ts +++ b/cli/src/commands/app/lint.ts @@ -9,6 +9,8 @@ import { GlobalOptions } from "../../types.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; import { loadRunnablesFromBackend } from "./raw_apps.ts"; +import { inferSchema } from "../../utils/metadata.ts"; +import { statedIsolation } from "./app.ts"; import { getFolderSuffix, hasFolderSuffix, @@ -42,12 +44,62 @@ function validateRawAppYaml(appData: any): { errors.push("Field 'summary' must be a string"); } + try { + statedIsolation(appData, true); + } catch (error: any) { + errors.push(error.message); + } + // Note: 'runnables' is no longer required in raw_app.yaml // Runnables can be stored in separate files in the backend folder return { errors, warnings }; } +const SQL_LANGUAGES = new Set([ + "postgresql", + "mysql", + "bigquery", + "snowflake", + "mssql", + "oracledb", + "duckdb", +]); + +// The viewer properties a `ctx` field can supply. The Inputs tab of the app +// editor hints on the same names (RawAppInlineScriptRunnable.svelte). +const VIEWER_IDENTITY_ARGS = ["username", "email", "groups"]; + +/** + * A SQL runnable argument named after a viewer property is most often a + * per-viewer filter. Left as a plain input it takes whatever the browser + * sends, so it has to be a `ctx` field for the server to fill it. + */ +export async function viewerIdentityInputWarnings( + runnables: Record, +): Promise { + const warnings: string[] = []; + for (const [id, runnable] of Object.entries(runnables)) { + const { language, content } = runnable?.inlineScript ?? {}; + if (!SQL_LANGUAGES.has(language) || typeof content !== "string") { + continue; + } + const { schema } = await inferSchema(language, content, {}, id); + for (const arg of VIEWER_IDENTITY_ARGS) { + const field = runnable.fields?.[arg]; + const serverFilled = field?.ctx !== undefined || + field?.value !== undefined; + if (arg in (schema?.properties ?? {}) && !serverFilled) { + warnings.push( + `Runnable '${id}': argument '${arg}' is a frontend input, so a viewer can send any value. ` + + `If it identifies the viewer, set 'fields: { ${arg}: { ctx: ${arg} } }' in backend/${id}.yaml (next to 'type: inline') so the server fills it.`, + ); + } + } + } + return warnings; +} + /** * Validates that runnables exist either in backend/*.yaml files or in raw_app.yaml */ @@ -77,6 +129,7 @@ async function validateRunnables( " - Or a 'runnables' field in raw_app.yaml (legacy format)", ); } else if (hasBackendRunnables) { + warnings.push(...await viewerIdentityInputWarnings(runnablesFromBackend)); log.info( colors.gray( ` Found ${ @@ -114,7 +167,7 @@ async function validateBuild( // Try to create a bundle - this will validate that all dependencies are in place const frameworks = detectFrameworks(appDir); - const entryFile = frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; + const entryFile = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = path.join(appDir, entryFile); await createBundle({ entryPoint, diff --git a/cli/src/commands/app/new.ts b/cli/src/commands/app/new.ts index b011965ae8..40cc2a4416 100644 --- a/cli/src/commands/app/new.ts +++ b/cli/src/commands/app/new.ts @@ -85,21 +85,6 @@ const app = mount(App, { target: document.getElementById("root")! }); export default app; `; -const appVue = ` - -`; - -const indexVue = `import { createApp } from 'vue' -import App from './App.vue' -import "./index.css"; - -createApp(App).mount('#root')`; - const indexCss = `body { margin: 0; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; @@ -171,21 +156,6 @@ const templates: Record = { "svelte": "^5.55.5", "windmill-client": "^1" } -}`, - }, - }, - vue: { - name: "Vue 3", - files: { - "/index.ts": indexVue, - "/App.vue": appVue, - "/index.css": indexCss, - "/package.json": `{ - "dependencies": { - "core-js": "3.26.1", - "vue": "3.5.13", - "windmill-client": "^1" - } }`, }, }, @@ -251,7 +221,7 @@ interface DataConfig { schema?: string; } -type FrameworkKey = "react19" | "react18" | "svelte5" | "vue"; +type FrameworkKey = "react19" | "react18" | "svelte5"; interface NewAppOptions extends GlobalOptions { /** App summary (short description). Skips the prompt when provided. */ @@ -384,7 +354,7 @@ async function newApp(opts: NewAppOptions) { } // Ask for framework (skipped if --framework is provided) - const VALID_FRAMEWORKS: FrameworkKey[] = ["react19", "react18", "svelte5", "vue"]; + const VALID_FRAMEWORKS: FrameworkKey[] = ["react19", "react18", "svelte5"]; let framework: string; if (opts.framework !== undefined) { if (!VALID_FRAMEWORKS.includes(opts.framework)) { @@ -403,7 +373,6 @@ async function newApp(opts: NewAppOptions) { { name: "React 19 (Recommended)", value: "react19" }, { name: "React 18", value: "react18" }, { name: "Svelte 5", value: "svelte5" }, - { name: "Vue 3", value: "vue" }, ], }); } @@ -900,7 +869,7 @@ const command = new Command() ) .option( "--framework ", - "Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode." + "Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode." ) .option( "--datatable ", diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index 36c8260e25..4c3d8d5215 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -20,13 +20,16 @@ import { deepEqual, readTextFile } from "../../utils/utils.ts"; import { type AppExecutionMode, + type AppIsolation, basePolicy, executionModeForPush, finalizeDerivedPolicy, + isolationUpToDate, markAccessFromPolicy, preserveOnBehalfOfFields, replaceInlineScripts, repopulateFields, + statedIsolation, } from "./app.ts"; import type { PermissionedAsContext } from "../../core/permissioned_as.ts"; import { buildPermissionedAsContext } from "../../core/permissioned_as.ts"; @@ -49,6 +52,8 @@ export interface AppFile { runnables?: any; custom_path?: string; public?: boolean; + sandbox?: boolean; + frontend_api?: string[]; summary: string; policy: Policy; data?: { @@ -402,6 +407,7 @@ export async function pushRawApp( } const appFilePath = localPath + "raw_app.yaml"; const localApp = (await yamlParseFile(appFilePath)) as AppFile; + const isolation = statedIsolation(localApp, true); // Load runnables from separate YAML files in the backend folder // Falls back to reading from raw_app.yaml if no separate files exist (backward compat) @@ -454,6 +460,7 @@ export async function pushRawApp( deployedPolicy, !!preserveFields.preserve_on_behalf_of, ), + isolation, ); const files = await collectAppFiles(localPath); @@ -461,7 +468,7 @@ export async function pushRawApp( log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`)); // Detect frameworks to determine entry point const frameworks = detectFrameworks(localPath); - const entryFile = frameworks.svelte || frameworks.vue + const entryFile = frameworks.svelte ? "index.ts" : "index.tsx"; const entryPoint = localPath + entryFile; @@ -485,14 +492,21 @@ export async function pushRawApp( // to `.raw_app.json`, not `.app.json`. The backend granular_acls // handler routes "raw_app" to the `app` table (where v2 raw apps actually // live) while still dispatching DeployedObject::RawApp for git-sync. - const { extra_perms: localPerms, ...localAppNoPerms } = localApp as AppFile & { + const { + extra_perms: localPerms, + sandbox: _sandbox, + frontend_api: _frontendApi, + ...localAppNoPerms + } = localApp as AppFile & { extra_perms?: Record; }; if (app) { // Check both metadata/runnables AND files for changes // Files need separate comparison because isSuperset only checks if local keys exist in remote - const metadataUpToDate = isSuperset({ ...localAppNoPerms, runnables }, app); + const metadataUpToDate = + isSuperset({ ...localAppNoPerms, runnables }, app) && + isolationUpToDate(isolation, deployedPolicy, true); const filesUpToDate = deepEqual(files, app.value?.files); if (metadataUpToDate && filesUpToDate) { log.info(colors.green(`App ${remotePath} is up to date`)); @@ -560,11 +574,12 @@ export async function generatingPolicy( path: string, executionMode: AppExecutionMode, base: Policy | undefined, + isolation?: AppIsolation, ) { log.info(colors.gray(`Generating fresh policy for app ${path}...`)); try { app.policy = await windmillUtils.updateRawAppPolicy(app.runnables, base); - finalizeDerivedPolicy(app.policy, executionMode); + finalizeDerivedPolicy(app.policy, executionMode, isolation, true); } catch (e) { log.error(colors.red(`Error generating policy for app ${path}: ${e}`)); throw e; diff --git a/cli/src/commands/init/template.ts b/cli/src/commands/init/template.ts index 8a433d69d4..540d8c7978 100644 --- a/cli/src/commands/init/template.ts +++ b/cli/src/commands/init/template.ts @@ -77,8 +77,8 @@ const WORKSPACE_CONFIG_SCHEMA = { */ export const CONFIG_REFERENCE: ConfigOption[] = [ // ── Core ────────────────────────────────────────────────────────────── - { name: "defaultTs", type: "string", enum: ["bun", "deno"], default: "bun", description: "Default TypeScript runtime for new scripts", - inlineComment: "bun or deno" }, + { name: "defaultTs", type: "string", enum: ["bun", "deno"], default: "bun", description: "TypeScript runtime of a plain .ts script (bun when absent)", + skipInTemplate: true }, { name: "includes", type: "array", items: { type: "string" }, default: '["f/**"]', description: "Glob patterns for files to include in sync", templateValue: '\n - "f/**"', inlineComment: 'use ** for all, f/** for folder-scoped' }, { name: "extraIncludes", type: "array", items: { type: "string" }, default: "[]", description: "Additional glob patterns merged with includes (useful in workspace overrides)", diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 54887028f0..5ba8888332 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -149,7 +149,11 @@ import { extractCurrentMapping, } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; -import { markAccessFromPolicy } from "../app/app.ts"; +import { + markAccessFromPolicy, + markIsolationFromPolicy, + stripEditorState, +} from "../app/app.ts"; import { APP_BACKEND_FOLDER, generateAppLocksInternal, @@ -1329,6 +1333,7 @@ export function ZipFSElement( log.error(`Failed to parse app.yaml at path: ${p}`); throw error; } + stripEditorState(app); let inlineScripts; try { inlineScripts = extractInlineScriptsForApps( @@ -1359,6 +1364,7 @@ export function ZipFSElement( } markAccessFromPolicy(app); + markIsolationFromPolicy(app, false); app.policy = undefined; yield { isDirectory: false, @@ -1377,6 +1383,7 @@ export function ZipFSElement( throw error; } markAccessFromPolicy(rawApp); + markIsolationFromPolicy(rawApp, true); // console.log("rawApp", rawApp); rawApp.policy = undefined; // custom_path is derived from the file path, don't store it @@ -2747,12 +2754,18 @@ export async function compareDynFSElement( } return o; } else { + let o: any; try { - return yamlParseContent(k, v); + o = yamlParseContent(k, v); } catch (error) { log.error(`Failed to parse YAML content at path: ${k}`); throw error; } + // A file tracked before the field was dropped still carries it. + if (path.basename(k) == "app.yaml") { + stripEditorState(o); + } + return o; } } diff --git a/cli/src/commands/user/user.ts b/cli/src/commands/user/user.ts index b5ed0e3196..99bea072a8 100644 --- a/cli/src/commands/user/user.ts +++ b/cli/src/commands/user/user.ts @@ -1,6 +1,8 @@ import { writeFile } from "node:fs/promises"; import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { mergeConfigWithConfigFile } from "../../core/conf.ts"; import { GlobalOptions, isSuperset, @@ -69,6 +71,32 @@ async function list(opts: GlobalOptions) { .render(); } +async function listWorkspaceUsers(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + opts = await mergeConfigWithConfigFile(opts); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const users = await wmill.listUsers({ workspace: workspace.workspaceId }); + + if (opts.json) { + console.log(JSON.stringify(users)); + } else { + new Table() + .header(["username", "email", "role"]) + .padding(2) + .border(true) + .body( + users.map((u) => [ + u.username, + u.email, + u.is_admin ? "admin" : u.operator ? "operator" : "developer", + ]) + ) + .render(); + } +} + function rdString() { return Math.random().toString(36).slice(2, 7); } @@ -518,6 +546,9 @@ export async function pushInstanceGroups( const command = new Command() .description("user related commands") .action(list as any) + .command("list", "List the users of the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(listWorkspaceUsers as any) .command("add", "Create a user") .arguments(" [password:string]") .option("--superadmin", "Specify to make the new user superadmin.") diff --git a/cli/src/commands/variable/variable.ts b/cli/src/commands/variable/variable.ts index 8963f376d2..88ad108a1a 100644 --- a/cli/src/commands/variable/variable.ts +++ b/cli/src/commands/variable/variable.ts @@ -107,7 +107,8 @@ export interface VariableFile { * plaintext value a user authored by hand. * * Mirrors the server guard (windmill-store/src/variables.rs): workspace - * ciphertext (AES-256-CBC, base64) is standard base64 decoding to a non-zero + * ciphertext (AES-256-GCM or legacy AES-256-CBC, base64) is standard base64 + * decoding to a non-zero * multiple of the 16-byte block size. External secret-backend markers * ($vault:/$aws_sm:/$azure_kv:) are stored verbatim too, so they count as * already-encrypted. This is a shape check only — it never decrypts. diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 455e876292..1ad0b69c1b 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -320,12 +320,6 @@ export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promis delete (conf as any)?.environments; delete conf?.git_branches; - if (conf?.defaultTs == undefined) { - log.warn( - "No defaultTs defined in your wmill.yaml. Using 'bun' as default." - ); - } - // Initialize global nonDottedPaths setting from config setNonDottedPaths(conf?.nonDottedPaths ?? false); @@ -376,7 +370,6 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< Required< Pick< SyncOptions, - | "defaultTs" | "includes" | "excludes" | "codebases" @@ -401,7 +394,6 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< > > > = { - defaultTs: "bun", includes: ["f/**"], excludes: [], codebases: [], diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 6b59042168..21aeb07f60 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.826.0"; +export const VERSION = "1.830.0"; diff --git a/cli/src/core/permissioned_as.ts b/cli/src/core/permissioned_as.ts index d0301373da..87ee37a8cb 100644 --- a/cli/src/core/permissioned_as.ts +++ b/cli/src/core/permissioned_as.ts @@ -34,10 +34,14 @@ export async function buildPermissionedAsContext( return undefined; } const user = await wmill.whoami({ workspace }); + // A superadmin acts as admin in every workspace, so the backend preserves + // ownership for them even when their own `usr` row there is not an admin. const userIsAdminOrDeployer = - user.is_admin || (user.groups ?? []).includes("wm_deployers"); + user.is_admin || + user.is_super_admin || + (user.groups ?? []).includes("wm_deployers"); log.debug( - `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` + `permissioned_as: user=${user.email}, is_admin=${user.is_admin}, is_super_admin=${user.is_super_admin}, groups=${JSON.stringify(user.groups)}, isAdminOrDeployer=${userIsAdminOrDeployer}` ); return { userCache: new Map(), diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 57e89f5ded..75dfa3b8da 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -413,7 +413,7 @@ export async function pushWorkspaceSettings( const localOperatorSettings = localSettings.operator_settings && { ...localSettings.operator_settings, }; - for (const key of ["builder_flows", "manage_schedules", "manage_triggers"] as const) { + for (const key of ["builder_flows", "builder_apps", "manage_schedules", "manage_triggers"] as const) { const remote = settings.operator_settings?.[key]; if (localOperatorSettings && localOperatorSettings[key] === undefined && remote !== undefined) { localOperatorSettings[key] = remote; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index 4aaf6d7e3c..0a880b05cf 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -34,6 +34,7 @@ export const SKILLS: SkillMetadata[] = [ { name: "resources", description: "MUST use when managing resources." }, { name: "write-workflow-as-code", description: "MUST use when writing or modifying Windmill Workflow-as-Code scripts using workflow, task, step, sleep, approvals, taskScript, taskFlow, task_script, or task_flow." }, { name: "write-pipeline", description: "MUST use when creating or modifying a data pipeline, a set of scripts marked `pipeline` and wired together by `on` / `materialize` annotations." }, + { name: "datatables", description: "MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions." }, { name: "cli-commands", description: "MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`." }, { name: "preview", description: "MUST use when opening the Windmill dev page / visual preview of a flow, script, or app. Triggers on words like preview, open, navigate to, visualize, see the flow/app/script, and after writing a flow/script/app for visual verification." }, ]; @@ -767,21 +768,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -837,6 +840,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -1096,6 +1107,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -1600,21 +1614,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1670,6 +1686,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -1929,6 +1953,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -2539,21 +2566,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -2609,6 +2638,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -2868,6 +2905,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -4434,10 +4474,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# \`\`retry\`\` re-runs the job when it fails, with the policy a schedule takes, +# e.g. \`\`{"constant": {"attempts": 3, "seconds": 10}}\`\`. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See \`\`run_script_by_path_async\`\` for \`\`retry\`\`. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -4610,6 +4654,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -5027,7 +5083,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -5113,12 +5171,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -5565,6 +5626,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` +## Array parameters + +Declare a list with a \`[]\` suffix on the element type (\`(varchar[])\`, +\`(int[])\`), or \`(array)\` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +\`PARSE_JSON(?)\`, and cast to \`ARRAY\` where a function expects one. + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +\`\`\` + +Or join on the elements with \`FLATTEN\`: + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +\`\`\` + ## Receiving an S3Object as a script parameter Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for @@ -5969,6 +6051,10 @@ can call one, not an agent used as a tool. - \`results.step_id.property\` - Access specific property from a previous step output only when that step result is in scope - \`flow_input.iter.value\` - Current iteration value inside a \`forloopflow\`; in a \`whileloopflow\` it is just the iteration index (a plain number, same as \`flow_input.iter.index\`) - \`flow_input.iter.index\` - Current loop index when inside a loop (\`forloopflow\` or \`whileloopflow\`) +- After a suspend step (approval), the next step's javascript expressions also get: + - \`resume\` - the last resume payload (the resume form values); with \`continue_on_disapprove_timeout\`, a disapproval or timeout yields \`{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }\` instead + - \`resumes\` - every resume payload, in arrival order + - \`approvers\` - a plain array of strings, one per resume: for a resume through the \`resume\`/\`cancel\` URL from \`getResumeUrls(approver)\`, the \`approver\` passed there if any; otherwise (including the \`approvalPage\` link) the resuming user's username, else \`"anonymous"\`, e.g. \`approvers[0]\`. It is NOT the \`{ resume_id, approver }\` objects stored in the flow status, so \`approvers[0].approver\` is always undefined ## Loop Structure Rules @@ -6280,7 +6366,7 @@ Reference a specific resource using \`$res:\` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \\u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \\u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} `, "raw-app": `--- name: raw-app @@ -6301,7 +6387,7 @@ You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. \`f/folder/my_app\` or \`u/username/my_app\` -3. **framework** — one of \`react19\` (recommended), \`react18\`, \`svelte5\`, \`vue\` +3. **framework** — one of \`react19\` (recommended), \`react18\`, \`svelte5\` **summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. @@ -6373,7 +6459,7 @@ my_app{{RAW_APP_SUFFIX}}/ ├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) ├── raw_app.yaml # App configuration (summary, path, data settings) ├── index.tsx # Frontend entry point -├── App.tsx # Main React/Svelte/Vue component +├── App.tsx # Main React/Svelte component ├── index.css # Styles ├── package.json # Frontend dependencies ├── wmill.ts # Auto-generated backend type definitions (DO NOT EDIT) @@ -6472,7 +6558,7 @@ data: - \`/
\` — Specific table in public schema - \`/:
\` — Table in specific schema -**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. \`roles\` records the role the app uses each datatable through; the app's code must pass the same role: \`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python. A datatable without an entry is used as its default role. +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. \`roles\` records the role the app uses each datatable through; the app's code must pass the same role: \`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python. A datatable without an entry is used as its default role, if it has one, plus the roles of whoever is using the app unless that default is \`admin\`. ## SQL Migrations (sql_to_apply/) @@ -6531,7 +6617,7 @@ For the rest, tell the user which command fits their intent and let them run it # Windmill Raw Apps -Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. +Raw apps let you build custom frontends with React or Svelte that connect to Windmill backend runnables and datatables. ## App shape @@ -6547,7 +6633,7 @@ A new app gets the setup Windmill's own new-app dialog gives one: a framework, t ### Framework -Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18 and Svelte 5 are for when they are asked for. These three are the only frameworks the raw app bundler builds. When the user asks to build or convert an app in another framework, write no code in it: say it is not available and offer these three instead. ### Data setup @@ -6577,7 +6663,7 @@ Then build what was chosen: the data table and schema become the app's \`data.da ### Entrypoint -The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte and Vue. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` / \`App.vue\` and keep the entrypoint as the mount shim. +The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` and keep the entrypoint as the mount shim. React (\`index.tsx\`): @@ -6589,7 +6675,7 @@ import App from './App' createRoot(document.getElementById('root')!).render() \`\`\` -Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. Vue (\`index.ts\`): \`createApp(App).mount('#root')\`. +Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. **Never replace the entrypoint with a bare component** (\`export default function App() { ... }\` and no mount call). A component that is defined but never mounted renders a blank screen with **no error thrown** — it never executes, so nothing reaches the console or the error overlay. If an app renders blank, check that the entrypoint still mounts \`App\` into \`#root\`. @@ -6662,16 +6748,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`@windmill-labs/chat-sdk\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. \`\`\`tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() \`\`\` -\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. +\`@windmill-labs/chat-sdk/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`@windmill-labs/chat-sdk/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -6755,6 +6841,18 @@ A draft app is reachable by nobody; deploying is what exposes it, and its backen Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. \`public: true\` in \`raw_app.yaml\` deploys the app for anonymous users, and \`guests: true\` for guests. Remove the line and the next push closes the app again. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are \`jobs:run\`, \`jobs:read\`, \`users:read\`, \`resources:read\`, \`variables:read\`, \`flow_conversations:read\` and \`flow_conversations:write\`; grant only the ones the frontend calls. +In \`raw_app.yaml\`: + +\`\`\`yaml +sandbox: true +frontend_api: [jobs:run, users:read] +\`\`\` + +A file with a \`sandbox\` key states both settings: \`sandbox: false\` turns isolation off, and a missing \`frontend_api\` means no scopes. A file without the key leaves the deployed app's isolation and scopes as they are, so removing \`sandbox: true\` does not unsandbox it. \`frontend_api\` needs \`sandbox: true\`; the push fails otherwise. + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. @@ -6766,6 +6864,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer 3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. 4. **Qualify table names**: an unqualified name means the \`public\` schema, so write every other table as \`schema.table\`, in table creation and queries alike. The app's \`data\` config sets the default datatable and schema its tables go in; use them consistently across runnables. 5. **Pass the role**: when the app's \`data.roles\` gives a data table a role, every \`wmill.datatable\` call on it passes that role (\`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python). The role only reaches what it was granted, so a query outside it fails with \`permission denied\`. +6. **Per-viewer rows**: a data table with no entry in \`data.roles\` is called with no role. Unless its default role is \`admin\`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as \`pg_has_role(current_user, '', 'member')\` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no \`data.roles\` entry. \`wmill datatable list\` lists the workspace's data tables. Create or change tables with a migration in \`sql_to_apply/\` (see "SQL Migrations" above), then add them to \`data.tables\` in \`raw_app.yaml\`. @@ -6819,7 +6918,7 @@ def main(user_id: str): 7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `, "triggers": `--- name: triggers @@ -7386,6 +7485,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with \`@task()\` or \`@task\`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or \`step()\` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. \`sleep()\` there waits in the job, and \`taskScript()\` / \`taskFlow()\` / \`waitForApproval()\` (\`task_script()\` / \`task_flow()\` / \`wait_for_approval()\`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: \`\`\`typescript @@ -7551,8 +7652,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -7596,6 +7698,9 @@ export function workflow(fn: (...args: any[]) => Promise) * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> @@ -7694,7 +7799,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -7780,12 +7887,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -7865,10 +7975,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake:///
\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +\`\`\`python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +\`\`\` + +\`\`\`ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake\`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch() +} +\`\`\` + +- Use \`INSERT INTO
SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first. +- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/
\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms. +- \`wmill.ducklake("")\` targets a non-default catalog; \`wmill.datatable("")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -7912,7 +8060,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. @@ -7976,6 +8124,53 @@ Node \`f/sales/orders_daily\` (runs when \`orders\` is produced, writes a rollup SELECT date_trunc('day', ts) AS day, count(*) AS n FROM ducklake.main.orders GROUP BY 1 \`\`\` +`, + "datatables": `--- +name: datatables +description: MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions. +--- + +# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, \`main\` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an \`up\` and an optional \`down\`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in \`BEGIN; ... END;\` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as \`CREATE INDEX CONCURRENTLY\`: such a migration can fail half applied. +- Give it a \`down\` that undoes the \`up\` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + +\`wmill datatable migrate new \` scaffolds \`migrations/datatable//_.up.sql\` and \`.down.sql\`, \`wmill datatable migrate up\` pushes the local files and applies what is pending, \`wmill datatable migrate down\` rolls back the most recent one and \`wmill datatable migrate status\` lists them. All take \`-d \` and default to \`main\`. + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (\`CREATE SCHEMA IF NOT EXISTS\`), and point what you are building at it. A raw app names its schema in its \`data\` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. \`admin\` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (\`wmill.datatable('main', { role: 'analyst' })\`, \`role='analyst'\` in Python, \`datatable://main?role=analyst\`) and holds that role alone, or names none and is resolved from the data table's default role; +- \`permission denied\` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to \`admin\` to get past it; +- a migration runs as the role its SQL declares on its first line, \`-- role \`, the same one in its \`down\`. One that declares none runs as \`admin\`, which only those who may use \`admin\` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. `, "cli-commands": `--- name: cli-commands @@ -8014,7 +8209,7 @@ app related commands - \`app dev [app_folder:string]\` - Start a development server for building apps with live reload and hot module replacement - \`--port \` - Port to run the dev server on (will find next available port if occupied) - \`--host \` - Host to bind the dev server to - - \`--entry \` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) + - \`--entry \` - Entry point file (default: index.ts for Svelte, index.tsx otherwise) - \`--no-open\` - Don't automatically open the browser - \`--recording\` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development - \`app lint [app_folder:string]\` - Lint a raw app folder to validate structure and buildability @@ -8025,7 +8220,7 @@ app related commands - \`app new\` - create a new raw app from a template - \`--summary \` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode. - \`--path \` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - - \`--framework \` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. + - \`--framework \` - Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode. - \`--datatable \` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - \`--schema \` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - \`--overwrite\` - Overwrite the target directory if it already exists, without prompting. @@ -8711,6 +8906,8 @@ user related commands **Subcommands:** +- \`user list\` - List the users of the workspace + - \`--json\` - Output as JSON (for piping to jq) - \`user add [password:string]\` - Create a user - \`--superadmin\` - Specify to make the new user superadmin. - \`--company \` - Specify to set the company of the new user. @@ -9061,7 +9258,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9072,7 +9270,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9160,7 +9359,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9171,7 +9371,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9244,7 +9445,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9255,7 +9457,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9365,7 +9568,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9376,7 +9580,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9528,7 +9733,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9539,7 +9745,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9697,7 +9904,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9708,7 +9916,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9823,7 +10032,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9834,7 +10044,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -9921,7 +10132,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -9932,7 +10144,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10013,7 +10226,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10024,7 +10238,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10111,7 +10326,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10122,7 +10338,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10262,7 +10479,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10273,7 +10491,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 @@ -10463,7 +10682,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -10474,7 +10694,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index 611268c27c..e3deabb6a7 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -189,6 +189,8 @@ const LANG_ANNOTATIONS: Partial> = { "py311", "py312", "py313", + "py314", + "py315", "sandbox", ], bun: ["npm", "nodejs", "native", "nobundling", "sandbox"], @@ -208,7 +210,7 @@ const LANG_ANNOTATIONS: Partial> = { * * `# py: ` is the exception the macro does not cover: the python * import parser reads it directly (`windmill-parser-py-imports`, alongside the - * `py310`..`py313` flags) to pick the interpreter, which changes what resolves. + * `py310`..`py315` flags) to pick the interpreter, which changes what resolves. */ export function hasLockAffectingAnnotation( scriptContent: string, diff --git a/cli/test/app_isolation_unit.test.ts b/cli/test/app_isolation_unit.test.ts new file mode 100644 index 0000000000..201c4a52a4 --- /dev/null +++ b/cli/test/app_isolation_unit.test.ts @@ -0,0 +1,60 @@ +import { expect, test } from "bun:test"; +import { + finalizeDerivedPolicy, + isolationUpToDate, + markIsolationFromPolicy, + statedIsolation, +} from "../src/commands/app/app.ts"; + +function pushed(file: any, deployed: any, rawApp = true) { + const policy = { ...deployed }; + finalizeDerivedPolicy(policy, "publisher", statedIsolation(file, rawApp), rawApp); + return policy; +} + +test("a pull writes isolation only when it is on, and a push deploys what was pulled", () => { + const off: any = { policy: { sandbox: false, frontend_sdk_scopes: ["jobs:run"] } }; + markIsolationFromPolicy(off, true); + expect("sandbox" in off).toBe(false); + expect("frontend_api" in off).toBe(false); + + const on: any = { policy: { sandbox: true, frontend_sdk_scopes: ["jobs:run"] } }; + markIsolationFromPolicy(on, true); + expect(on.sandbox).toBe(true); + expect(on.frontend_api).toEqual(["jobs:run"]); + // Into a workspace where the app does not exist yet. + const fresh = pushed(on, undefined); + expect(fresh.sandbox).toBe(true); + expect(fresh.frontend_sdk_scopes).toEqual(["jobs:run"]); +}); + +test("the file wins when it states sandbox, the deployed policy carries over when it does not", () => { + const deployed = { sandbox: true, frontend_sdk_scopes: ["jobs:run"] }; + + const silent = pushed({}, deployed); + expect(silent.sandbox).toBe(true); + expect(silent.frontend_sdk_scopes).toEqual(["jobs:run"]); + expect(isolationUpToDate(statedIsolation({}, true), deployed, true)).toBe(true); + + // A file with the key states the scopes too: none here. + const narrowed = pushed({ sandbox: true }, deployed); + expect(narrowed.sandbox).toBe(true); + expect(narrowed.frontend_sdk_scopes).toBeUndefined(); + expect(isolationUpToDate(statedIsolation({ sandbox: true }, true), deployed, true)).toBe(false); + + const opened = pushed({ sandbox: false }, deployed); + expect(opened.sandbox).toBe(false); + expect(opened.frontend_sdk_scopes).toBeUndefined(); +}); + +test("scopes are refused without sandbox, outside the allowlist, or repeated", () => { + expect(() => statedIsolation({ frontend_api: ["jobs:run"] }, true)).toThrow("requires 'sandbox: true'"); + expect(() => statedIsolation({ sandbox: false, frontend_api: ["jobs:run"] }, true)).toThrow( + "requires 'sandbox: true'", + ); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["apps:write"] }, true)).toThrow("Invalid"); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["jobs:run", "jobs:run"] }, true)).toThrow( + "Duplicate", + ); + expect(() => statedIsolation({ sandbox: true, frontend_api: ["jobs:run"] }, false)).toThrow("full-code"); +}); diff --git a/cli/test/app_lint_viewer_identity_unit.test.ts b/cli/test/app_lint_viewer_identity_unit.test.ts new file mode 100644 index 0000000000..f328ffe4e2 --- /dev/null +++ b/cli/test/app_lint_viewer_identity_unit.test.ts @@ -0,0 +1,24 @@ +import { expect, test } from "bun:test"; +// app.ts first: it is the entry the CLI loads lint.ts through, and loading +// lint.ts ahead of it trips their import cycle. +import "../src/commands/app/app.ts"; +import { viewerIdentityInputWarnings } from "../src/commands/app/lint.ts"; + +const sql = (content: string, fields?: Record) => ({ + type: "inline", + inlineScript: { language: "duckdb", content }, + ...(fields ? { fields } : {}), +}); + +test("app lint warns on a SQL viewer-identity argument the server does not fill", async () => { + const content = + "-- $username (text)\n-- $limit (int)\nSELECT * FROM orders WHERE owner = $username LIMIT $limit"; + + const warnings = await viewerIdentityInputWarnings({ + open: sql(content), + filled: sql(content, { username: { ctx: "username" } }), + }); + + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("Runnable 'open': argument 'username'"); +}); diff --git a/cli/test/init_no_git_sync.test.ts b/cli/test/init_no_git_sync.test.ts index d16c9e771f..189e82562f 100644 --- a/cli/test/init_no_git_sync.test.ts +++ b/cli/test/init_no_git_sync.test.ts @@ -56,8 +56,6 @@ test("Init: createWorkspaceProfile includes defaults when no repositories exist" expect(Array.isArray(workspaceProfile.excludes)).toBeTruthy(); expect(workspaceProfile.excludes.length).toEqual(0); - expect(workspaceProfile.defaultTs).toEqual('bun'); - console.log('Workspace profile correctly includes default sync settings when no repositories exist'); }); @@ -74,8 +72,6 @@ test("Init: verify DEFAULT_SYNC_OPTIONS has expected values", () => { expect(Array.isArray(DEFAULT_SYNC_OPTIONS.excludes)).toBeTruthy(); expect(DEFAULT_SYNC_OPTIONS.excludes.length).toEqual(0); - expect(DEFAULT_SYNC_OPTIONS.defaultTs).toEqual('bun'); - console.log('DEFAULT_SYNC_OPTIONS has expected values'); }); @@ -178,7 +174,7 @@ test.skipIf(shouldSkipOnCI())("Init: --use-default bypasses backend settings che // Should have default settings, not backend settings expect(wmillYaml).toContain("includes:\n - f/**"); - expect(wmillYaml).toContain("defaultTs: bun"); + expect(wmillYaml).not.toContain("defaultTs"); // Should NOT have backend-specific settings expect(wmillYaml.includes("f/should-be-ignored/**")).toEqual(false); diff --git a/cli/test/init_template_unit.test.ts b/cli/test/init_template_unit.test.ts index 227fc271a2..7070de63c8 100644 --- a/cli/test/init_template_unit.test.ts +++ b/cli/test/init_template_unit.test.ts @@ -90,7 +90,6 @@ describe("generateCommentedTemplate", () => { test("default values match expected defaults", () => { const config = parse(generateCommentedTemplate("main")); - expect(config.defaultTs).toBe("bun"); expect(config.skipSecrets).toBe(true); expect(config.nonDottedPaths).toBe(true); expect(config.codebases).toEqual([]); diff --git a/cli/test/permissioned_as_context_unit.test.ts b/cli/test/permissioned_as_context_unit.test.ts new file mode 100644 index 0000000000..33a24bf3a1 --- /dev/null +++ b/cli/test/permissioned_as_context_unit.test.ts @@ -0,0 +1,27 @@ +/** + * Regression guard: a superadmin whose `usr` row in the workspace is not an + * admin still preserves ownership on the backend, so the CLI must not refuse + * their push as if it would reassign every item to them. + */ + +import { expect, test } from "bun:test"; +import { mockServices } from "./mock_services.ts"; + +mockServices({ + whoami: async () => ({ + email: "super@windmill.dev", + username: "super", + is_admin: false, + is_super_admin: true, + groups: [], + }), +}); + +const { buildPermissionedAsContext } = await import( + "../src/core/permissioned_as.ts" +); + +test("a superadmin with a non-admin membership counts as able to preserve", async () => { + const ctx = await buildPermissionedAsContext("w", "v1"); + expect(ctx?.userIsAdminOrDeployer).toBe(true); +}); diff --git a/cli/wmill.schema.json b/cli/wmill.schema.json index 022156e5c9..145a75fd26 100644 --- a/cli/wmill.schema.json +++ b/cli/wmill.schema.json @@ -10,7 +10,7 @@ "bun", "deno" ], - "description": "Default TypeScript runtime for new scripts" + "description": "TypeScript runtime of a plain .ts script (bun when absent)" }, "includes": { "type": "array", diff --git a/docker/DockerfileExtra b/docker/DockerfileExtra index ace382528b..1d8d0003da 100644 --- a/docker/DockerfileExtra +++ b/docker/DockerfileExtra @@ -71,7 +71,7 @@ ENV XDG_CACHE_HOME=/pyls/.cache RUN uv pip install --system --break-system-packages pipenv tornado python-lsp-jsonrpc ruff==0.16.0 Cython # Install Node-based language servers -RUN npm install -g diagnostic-languageserver pyright +RUN npm install -g pyright # Setup LSP working directory WORKDIR /pyls diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 8ef6be78bb..c3a82c8312 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,6 +1,9 @@ ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill:dev FROM ${BASE_IMAGE} +# Deno +COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno + # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.97.0 /usr/local/rustup /usr/local/rustup @@ -12,7 +15,9 @@ RUN RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo /usr/local/cargo/b # ansible there leaves its venv interpreter as a dangling symlink at runtime, so every # ansible-* executable fails with ENOENT ("ansible-galaxy not found"). Pin the tool's # interpreter to a persistent image path so the install stays self-contained. -RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true +# The version is explicit because uv otherwise downloads its newest stable Python, which +# ansible-core may not support yet: the install succeeds and every executable fails on import. +RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install --python 3.14 ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true # C# RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 88ed619c19..bb83195f62 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -21,6 +21,9 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ FROM ${BASE_IMAGE} +# Deno +COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno + # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.97.0 /usr/local/rustup /usr/local/rustup @@ -32,7 +35,9 @@ RUN RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo /usr/local/cargo/b # ansible there leaves its venv interpreter as a dangling symlink at runtime, so every # ansible-* executable fails with ENOENT ("ansible-galaxy not found"). Pin the tool's # interpreter to a persistent image path so the install stays self-contained. -RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true +# The version is explicit because uv otherwise downloads its newest stable Python, which +# ansible-core may not support yet: the install succeeds and every executable fails on import. +RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install --python 3.14 ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true # dotnet SDK RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ && chmod +x dotnet-install.sh \ diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 45986bbd5f..a37ced0eca 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -58,7 +58,7 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) # --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 5ac24f5c1d..687b1506ce 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -58,7 +58,7 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) # --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run diff --git a/docker/test_windmill_extra.ts b/docker/test_windmill_extra.ts index 5fd1a74108..c5884c2dd4 100644 --- a/docker/test_windmill_extra.ts +++ b/docker/test_windmill_extra.ts @@ -121,6 +121,60 @@ async function testLspWebSocket(): Promise { }) } +async function testLspShellDiagnostics(): Promise { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + ws.close() + reject(new Error('no shellcheck diagnostics within 10s')) + }, 10000) + + const ws = new WebSocket(`ws://${HOST}:${LSP_PORT}/ws/diagnostic`) + const send = (msg: object) => ws.send(JSON.stringify({ jsonrpc: '2.0', ...msg })) + + ws.onopen = () => { + // Editors still send a linter definition; the server's own configuration + // applies, so the diagnostics below have to come from shellcheck. + send({ + id: 1, + method: 'initialize', + params: { + capabilities: {}, + initializationOptions: { + linters: { shellcheck: { command: 'true', sourceName: 'shellcheck' } }, + filetypes: { shell: 'shellcheck' } + } + } + }) + send({ + method: 'textDocument/didOpen', + params: { + textDocument: { + uri: 'file:///tmp/monaco/test.sh', + languageId: 'shell', + version: 1, + text: '#!/bin/bash\necho "$unassigned"\n' + } + } + }) + } + + ws.onmessage = (event) => { + const msg = JSON.parse(event.data.toString()) + if (msg.method !== 'textDocument/publishDiagnostics') return + clearTimeout(timeout) + ws.close() + const codes = msg.params.diagnostics.map((d: { code: string }) => d.code) + if (codes.includes('SC2154')) resolve() + else reject(new Error(`expected SC2154, got ${JSON.stringify(msg.params.diagnostics)}`)) + } + + ws.onerror = () => { + clearTimeout(timeout) + reject(new Error('LSP WebSocket connection failed')) + } + }) +} + async function runLspTests(): Promise { log('\n=== LSP Service Tests (Port ' + LSP_PORT + ') ===') @@ -132,6 +186,7 @@ async function runLspTests(): Promise { let passed = true passed = (await runTest('LSP health check', testLspHealth)) && passed passed = (await runTest('LSP WebSocket connection (pyright)', testLspWebSocket)) && passed + passed = (await runTest('LSP shell diagnostics', testLspShellDiagnostics)) && passed return passed } diff --git a/docs/datatable-viewer-roles.md b/docs/datatable-viewer-roles.md new file mode 100644 index 0000000000..9855e4f97a --- /dev/null +++ b/docs/datatable-viewer-roles.md @@ -0,0 +1,93 @@ +# Data table connections and the viewer's roles + +A data table under roles (Enterprise Edition) resolves a connection that names a role to that +role alone. A connection that names none is resolved from the data table's default role: + +| Default role | A connection naming no role holds | +|---|---| +| `admin` (until changed) | `admin`, as before viewer roles existed | +| another role | that role when the job's own identity may use it, plus **every role the job's viewer is a tenant of** | +| none (`NO_DEFAULT_DATATABLE_ROLE`) | the viewer's roles only | + +A connection left with no role at all is refused. Postgres row-level security can then answer +per viewer from a backend runnable that runs as the app's publisher, with nothing written in the +runnable. + +Policies test membership, which is true for a role's own login too, so one policy serves a named +connection and an unnamed one: + +```sql +CREATE POLICY emea ON orders FOR SELECT + USING (region = 'emea' AND pg_has_role(current_user, 'sales_emea', 'member')); +``` + +A policy names its role as text, so renaming the role breaks it. + +## Code + +| Where | What | +|---|---| +| `windmill-common/src/workspaces.rs` | `DatatableAccess`, `get_datatable_resource_from_db` | +| `windmill-common/src/datatable_roles_ee.rs` | `resolve_datatable_role_connection`, `viewer_role_ids`, `ensure_role_set_login`, `retire_role_set_logins_of`, `sweep_unused_role_set_logins` | +| `windmill-api-jobs/src/execution.rs` | `run_end_user_email`, where a run's viewer is recorded | +| `frontend/.../workspaceSettings/DataTablePermissionsButton.svelte` | the roles drawer, which warns while the default is `admin` | + +## What holds it together + +- **A default of `admin` is not a floor.** `admin` owns the tables and bypasses row-level + security, so nothing could be added to it, and it is the default until someone changes it. + Such a data table resolves exactly as it did before viewer roles existed. Nothing ever sets a + default back to `admin`: removing or deleting the default role leaves none. +- **The default role is tenant-checked against the job's own identity**, the publisher for a + publisher-mode app, and held only when that passes. The viewer's roles are added either way. +- **No role at all is a refusal, never an empty answer.** There is no login that would read + nothing: what a role sees is up to each table's grants and policies. A job without a recorded + viewer (anonymous app viewer, schedule, trigger, a tool job started by an AI agent step), or + whose viewer is not a member of the governing workspace, therefore needs a default it may use + or a named role. +- **A set login owns what it creates.** Postgres gives a new object to `current_user`, and + default privileges go by the creating role, so a table created over a connection that holds + several roles belongs to that set's login: no other role or viewer reaches it, and it passes to + `custom_instance_user` when the login is swept. Unnamed connections are for rows. To create an + object, name a role or run `SET ROLE ` first, which a set login may do for any role it + holds. Migrations are not affected: one that declares no role runs as `admin`. +- **The viewer is `job_perms.end_user_email`, and nothing else.** The server records it when the + run starts: the session behind an app request, the caller of a direct run or preview, and for a + job started with another job's token, that job's viewer. No argument reaches it. +- **A cached result belongs to one viewer.** `cached_result_path` hashes the run's recorded viewer + into the key, since a cache hit returns before any connection is resolved. Runs with no viewer + recorded keep sharing one entry. +- **Only a tenant list naming the viewer counts.** A workspace admin may *use* every role, + `admin` included. The viewer's roles are those whose tenants name them (user, group, folder, + `*`), and never `admin`. +- **One role is that role's login. Several are a login per distinct set.** `wm_set_` is a + member of exactly those roles, with `INHERIT`; the hash is over their catalog ids, so a rename + moves nothing and a change in someone's groups resolves them to another login instead of + editing one. Its privileges are its memberships, so `SET ROLE` only narrows and `RESET ROLE` + comes back to it. The prefix is reserved in `validate_role_name`. +- **Resolving stays cheap where no viewer could add anything.** The viewer is only looked up when + some role other than the default and `admin` has tenants on that data table. +- **A `datatable_role_set` row with a password names a login that can authenticate with it.** The + row is written after the login exists and removed, in its own commit, before the login is + dropped or made `NOLOGIN`. A login without a row is taken back with a new password on next use, + and so is one whose row has an empty password: that is what a sweep leaves when it could + neither drop a login nor let it back in, so the next sweep still finds it. +- **Set logins are bounded by use, not by the number of role combinations.** One exists only for + a set some viewer actually connected with, so there are at most as many as (viewer, data table) + pairs. `last_used_at` is recorded at most once a day, and `sweep_unused_role_set_logins`, run + from the monitor loop, drops those unused for 30 days; a dropped one is made again on demand. + The sweep goes by this instance's rows, never by the cluster's `wm_set_*` roles: a cluster can + serve several Windmill instances. +- **Disabling or dropping a role retires every set login holding it.** `NOLOGIN` on a role does + not reach a login that is a member of it. +- **A worker or server that predates this connects as the default role alone**, so a mixed fleet + shows a viewer fewer rows, never more. It refuses a data table with no default role, whose + marker it reads as a role the data table lacks. + +## Trust + +A job that names no role runs its author's code with the viewer's reach into the data table, the +same trust as an app in viewer execution mode or a script someone else wrote and you run. Whoever +can deploy a publisher-mode app, or a script that others run, can therefore read through the +non-`admin` roles of the people who use it. Naming a role in the code is how an author pins the +connection to one role instead. diff --git a/docs/operator-builder-rights.md b/docs/operator-builder-rights.md index 4db2a69511..4f844beebf 100644 --- a/docs/operator-builder-rights.md +++ b/docs/operator-builder-rights.md @@ -1,13 +1,25 @@ # Operator builder rights -`operator_settings.builder_flows` lets every operator of a workspace compose flows out of runnables -that already exist. It does not make them authors: the boundary the operator role draws is -**authoring code and running arbitrary code**, and this does not move it. +Two workspace settings, `operator_settings.builder_flows` and `operator_settings.builder_apps`, +that let every operator of that workspace compose flows and full-code apps out of runnables that +already exist. Neither lets them author backend code or run arbitrary code on a worker, the +boundary the operator role draws. The app right does let them write the app's frontend code, +which runs in the viewer's browser and is why that right rests on sandbox isolation. -It is a write right, unlike the visibility flags beside it, and unlike the withdrawable rights in -`docs/operator-write-rights.md` it is granted on request and costs a seat. Read it with -`windmill_common::workspaces::operator_can_build_flows` (60s cache, shared with the withdrawable -rights) and gate a write with `check_operator_can_build_flows`. +They are granted **independently**, because the two are not equally verifiable. A composed flow is +checked in full server-side (`check_flow_is_composition_only` refuses everything carrying code). +A full-code app is a browser-built bundle no server-side check can read, and rests instead on +forced `sandbox` isolation plus the viewer consent prompt. An admin may reasonably want the first +without the second, so **never gate on "either"**: a gate that does authorizes the kind its +workspace never granted, which is the whole point of the split. `BuilderKind` exists to make each +call site say which one it means. + +These are write rights, unlike the visibility flags beside them, and unlike the withdrawable +rights in `docs/operator-write-rights.md` they are granted on request and cost a seat. Read them +with `windmill_common::workspaces::operator_builder_rights` (60s cache, shared with the +withdrawable rights), which returns an `OperatorBuilderRights { flows, apps }`, and gate a write +with `check_operator_can_build`, naming the `BuilderKind`. `.any()` is only for surfaces that are +genuinely not per-kind: the seat a membership costs, and listing drafts. ## What the check has to cover @@ -49,25 +61,131 @@ Call it on every write **and** every preview: `run_preview_flow_job` and it the way to run what the write path refuses. A flow draft goes through `validate_operator_flow` too: a developer who loads a builder's draft in the editor runs its code as themselves. +The same reasoning applies to a builder-authored app, with one extra step. `execute_component` +resolves the runnable it runs on the root handle, so `validate_operator_composed_app` checks every +referenced path under the caller's RLS and refuses hub ones. For a script it checks the version a +run would pick: grants are per version, and a path reused after its versions were archived starts +with none while the archived ones keep theirs. But it has to check **two** surfaces, +because they are not the same list: the policy's `script/` and `flow/` triggerables, +and the by-path entries of the app value's `runnables` map, which is what the editor resolves a +`runnable_id` against and sends. Read only that map, and refuse an entry whose `type`, `path` or +`runType` is not text. + +Reading a triggerable key is not a `split_once(':')`: `execute_component` looks up +`format!("{component}:{path}")` with an unrestricted component string, so `a:b:script/x` resolves +at run time for `component = "a:b"`. Every colon is a possible split, so the check validates every +suffix that parses as a runnable rather than guessing which one a request will use. + +What makes those checks bind is that **`ExecutionMode::Viewer` is refused for a builder app**. In +Viewer mode `execute_component` falls back to a default triggerable for any `script/`/`flow/` +path, so the policy stops being the list of what the app may invoke, and the job runs as the +*viewer*: an admin who merely opened the app would run anything in the workspace as themselves. +`Publisher`, `Guest` and `Anonymous` have no such fallback; all three run as the publisher. If you ever relax the Viewer refusal, the +deploy-time path checks above stop being an authorization boundary. + +## The two raw-app deploy paths are not equivalent + +`create_app_raw` / `update_app_raw` are multipart: the browser already built the bundle, and +nothing server-side compiles anything. Builders use these. + +`create_app_raw_source` / `update_app_raw_source` push a bundler CLI over caller-supplied `files` +as a job **on a worker**, which is arbitrary code execution and is why they already require +`jobs:run` on top of `apps:write`. They stay closed to operators, builder rights or not. Do not +"tidy" the exception away: it costs builders nothing, because the browser and the CLI both bundle +locally and deploy through the multipart endpoints. + +A builder-authored app is forced to `policy.sandbox = true`. That is what makes it safe to let an +operator publish a bundle nobody reviewed: without it the bundle runs same-origin with each +viewer's Windmill session. The same check refuses a `rawscript/` key in either triggerables +map, since that key is the deployed app's authorization to run caller-supplied `raw_code` hashing +to it. It reads the value as stored, NUL escapes stripped: a stripped NUL can turn a key it +ignored into `inlineScript`. + +`update_app` takes a builder too, but settings only: the deploy panel saves a deployed full-code +app's access and frontend scopes through it with no value. It reaches the same check, so a builder +can no more switch the sandbox off there than on deploy. + +A builder's **draft** goes through the same `check_operator_composed_app` as its deploy, with +`BuilderAppWrite::Draft`: an admin may open it in the editor or deploy it as it stands, so any +rule added for deploys must hold for drafts too. Where the two differ, `write` decides: a deploy +fills in an omitted sandbox or mode, since it stores the policy it checked, while a draft skips +`triggerables_v2` and must not name another user's run identity. Every frontend writer of a +builder draft must conform the policy first with `conformBuilderAppPolicy`, since a policy loaded +from someone else's app breaks these rules: the editor and the AI chat do. The AI Sessions raw-app +editor, open to operators only behind a per-browser opt-in, does not, so its autosaves are refused. + +The raw-app editor previews an app isolated only when its policy has `sandbox`, and otherwise runs +the bundle same-origin with the session of whoever opens it. A builder's draft is therefore +held to the same rule as its deploy: `update_draft` refuses one whose `policy.sandbox` is not +`true`, so whatever an admin opens of an operator's, deployed or drafted, previews isolated. +Turning the sandbox off in their own editor stays that admin's explicit choice. It also refuses +a draft naming a runnable the builder cannot read, or a hub one, like the deploy: the editor +preview runs them as whoever opens it, and admits any path for an admin. That covers the +policy's legacy `triggerables` too, `rawscript/` keys included: every deploy of a draft +carries it through untouched, so an admin deploying the draft would publish it. Its +`triggerables_v2` is not checked: every deploy of a draft rebuilds it from the runnables, the +preview never reads it, and a copy gone stale since the last deploy would otherwise block +autosave. `Viewer` mode is refused in a draft as on deploy: the deploy panel shows it as +members-only, so an admin deploying the draft would publish it unawares. The builder's editor +switches it to `Publisher` on load, since that panel offers no way out of it. A draft naming a +run identity other than its builder is refused for the same reason: a builder's own deploy resets +it, but an admin's one-click draft deploy preserves it without showing it. The builder's editor +drops another user's identity on load, which an app that user deployed carries. A builder in +`wm_deployers` is the exception: like any member, their deploy may keep another identity, which +was judged acceptable for operators in that group (WIN-2536) and is the same for composed flows. + +Reading a triggerable key at every colon copies it once per colon, so the check refuses a key +over 512 bytes before splitting: a quadratic walk otherwise lets a sub-megabyte key allocate +gigabytes. A 255-character path plus a component name stays well under it. + ## Billing -An operator of a builder workspace consumes a full author seat: composing deployable artifacts -makes them an author, and there is no half-author. `consumes_operator_seat` is the seat-role -helper; the EE counting queries share `OPERATOR_SEAT_SQL` so the displayed, enforced and reported -numbers agree. The one exception is `get_user_usage` in `stats_ee.rs`: it is a compile-checked -`query!`, which cannot interpolate the constant, so it spells the predicate out. Change both -together. +An operator of a builder workspace consumes a full author seat. This is the one place that reads +"either right", and deliberately so: either one makes an operator an author of deployable +artifacts, and there is no half-author, so granting both costs no more than granting one. Do not +introduce per-right pricing — it turns one boolean into a capability matrix that billing, the cap +check and the displayed count would each have to agree on. -On cloud, `billable_seats` applies the same rule; see its doc for the out-of-repo invoice it must -match. +`consumes_operator_seat` is the seat-role helper; the EE counting queries share +`OPERATOR_SEAT_SQL` so the displayed, enforced and reported numbers agree. The one exception is +`get_user_usage` in `stats_ee.rs`: it is a compile-checked `query!`, which cannot interpolate the +constant, so it spells the predicate out. Change both together. On cloud, `billable_seats` applies +the same rule; see its doc for the out-of-repo invoice it must match. -Granting the right runs `check_seat_cap_for_operator_builder`, which prices the change by counting -seats twice rather than by counting the workspace's operators: an operator who already authors -elsewhere must not be charged again, so re-saving settings that already have the right on is a -zero delta and never blocks. +Granting the first right runs `check_seat_cap_for_operator_builder`, which prices the change by +counting seats twice rather than by counting the workspace's operators: an operator who already +authors elsewhere must not be charged again. Adding the second right later, or re-saving settings +that already have one on, is a zero delta and never blocks. ## Accepted risks +- A builder raw app may declare `frontend_sdk_scopes`, and `mint_raw_app_sdk_token` mints as the + *viewer*. A consenting admin therefore hands the bundle a 12h admin-identity token within the + curated scope list. The viewer consent prompt is the gate, shown by the editor preview as well + as the deployed app, before any token is minted. The lever, if this is ever revisited, + is dropping `variables:read` / `resources:read` / `jobs:run` from `FRONTEND_SDK_ALLOWED_SCOPES` + for builder apps. +- The raw-app editor preview runs an app's runnables as whoever has the editor open, where the + deployed app runs them as its publisher. An admin opening a builder's app in the editor lets its + frontend code run, with the admin's rights, any runnable the builder can read. Running as the + draft's author instead would need a preview that ignores unsaved edits to the runnable list. +- A builder may make an app public (`ExecutionMode::Anonymous`): anyone with the URL runs the + runnables its policy names, as the builder. Refusing it is the workspace's call, through the + `RestrictAnonymousAppDeployment` protection rule. A `Guest` app is the same with a login in + front: anyone the instance's identity provider admits, member or not, once guest access is + enabled. `RestrictGuestAppDeployment` refuses it. +- A builder raw app may set `allow_user_resources` on a runnable's inputs, and the bundle picks the + `$res:` path it sends. `build_args` resolves that path as the *viewer*, with no prompt, into a + job that runs as the builder. Getting a value out takes a guessable path and a readable script + that returns, logs or sends its input. Developers can do the same; the lever is refusing a + non-empty `allow_user_resources` (and `s3_inputs[].allow_user_resources`) for builders. +- A builder's delete or settings-only update reads the app's kind before the write, not under a + lock held through it. An admin converting that app to low-code in between lets the builder's + request land on the low-code app; the builder already has write access to its path. +- A builder's draft carries no run identity other than the builder's own, and deploying one that + carries none makes the deployer its run identity, as any deploy does. An admin deploying such a + draft, from the editor or Compare & Deploy, therefore publishes the builder's code running as + that admin; the deploy is the admin's explicit act, as it is for a developer's draft. - All-or-nothing per workspace: there is no per-user builder role. - `operator_settings` is git-synced, so a pull can flip every operator's class in a workspace and the billed seat count with it. diff --git a/docs/pipeline-local-dev.md b/docs/pipeline-local-dev.md index e05acec9f0..fb8d49b296 100644 --- a/docs/pipeline-local-dev.md +++ b/docs/pipeline-local-dev.md @@ -69,7 +69,7 @@ makes the client own the whole cascade so the backend dispatcher never double-fi - `frontend/src/lib/components/assets/AssetGraph/cascadeRun.ts` — reusable run primitives (`makeLaunch`, `makeWaitJobTerminal`, `runDownstreamCascade`, `runBoundedCascade`) wrapping `cascadeOrchestrator` + `graphTraversal`. -- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/pipeline_dev/+page.svelte`. +- `frontend/src/lib/components/assets/AssetGraph/PipelineDevView.svelte` + `frontend/src/routes/(styled)/pipeline_dev/+page.svelte`. **Modified** - `cli/src/commands/pipeline/pipeline.ts` — `--local` on `show`/`run`; `show` split into graph diff --git a/flake.nix b/flake.nix index f005644e3d..2a1d2699c8 100644 --- a/flake.nix +++ b/flake.nix @@ -78,15 +78,16 @@ ]; # --------------------------------------------------------------- - # Prebuilt V8 binary (must match version in Cargo.toml) + # Prebuilt V8 binary (must match the `v8` pin in backend/Cargo.toml). + # The simdutf variant: deno_core enables that feature of the v8 crate. # --------------------------------------------------------------- rustyV8Archive = let - version = "137.1.0"; + version = "150.4.0"; target = stdenv.hostPlatform.rust.rustcTarget; sha256 = { x86_64-linux = - "sha256-Tiscfy2bzYGR3s0T+SC1IB3xWvTVpVcSEdjq3MCRoRw="; + "sha256-9IdiyhDR8fxgWkQcWuQw7Izh6egPFNePvELLh4wwtHY="; aarch64-linux = lib.fakeHash; x86_64-darwin = lib.fakeHash; aarch64-darwin = lib.fakeHash; @@ -94,7 +95,7 @@ in pkgs.fetchurl { name = "librusty_v8-${version}"; url = - "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz"; + "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_simdutf_release_${target}.a.gz"; inherit sha256; }; diff --git a/frontend/e2e/global-setup.ts b/frontend/e2e/global-setup.ts index db27741a5a..6d10c53d92 100644 --- a/frontend/e2e/global-setup.ts +++ b/frontend/e2e/global-setup.ts @@ -47,18 +47,10 @@ async function globalSetup(config: FullConfig) { await page.waitForURL('**/') if (!process.env.SKIP_HUB_SYNC) { - await page.locator('#home-search-input').waitFor({ state: 'visible' }) - await page.locator('#home-search-input').fill('u/admin/hub_sync') - - const hubSyncBtn = page.locator('text=u/admin/hub_sync').first() - await hubSyncBtn.waitFor({ state: 'visible' }) - await hubSyncBtn.click() - - const runBtn = page.locator('#run-form-run-button') - await runBtn.waitFor({ state: 'visible' }) - await runBtn.click() - await page.waitForURL(/\/run\/.+/) - await page.locator('text=Success').waitFor({ timeout: 30000, state: 'visible' }) + const res = await page.request.post(`${baseURL}/api/settings/sync_cached_resource_types`) + if (!res.ok()) { + throw new Error(`Resource type sync failed: ${await res.text()}`) + } } // Save the authenticated state await context.storageState({ path: './e2e/auth.json' }) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e8ea3dece4..6130364ff9 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.830.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.830.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -95,19 +95,19 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.753.0", + "windmill-parser-wasm-asset": "1.830.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", - "windmill-parser-wasm-py": "1.782.0", + "windmill-parser-wasm-py": "1.830.0", "windmill-parser-wasm-r": "1.668.1", "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.695.0", - "windmill-parser-wasm-wac": "1.668.6", + "windmill-parser-wasm-wac": "1.830.0", "windmill-parser-wasm-yaml": "1.770.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "1.8.2", @@ -14644,9 +14644,9 @@ } }, "node_modules/windmill-parser-wasm-asset": { - "version": "1.753.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.753.0.tgz", - "integrity": "sha512-zpJhjvcU8EWRoOJzas/nRGKjGdQnvzeB9GOxP+Mdmnk8BFk3uehsmHS2Krxyjo36fUrCHqobbnffEqf0g3LIGg==" + "version": "1.830.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.830.0.tgz", + "integrity": "sha512-q4HSaMOd5cF8pmfPcEGiGeQbO8vhtv89us7zbO12IlgpxpWT8ojvw66J75y4yUt1PaEuaoztSaX8Gkxxn/6MXw==" }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", @@ -14674,9 +14674,9 @@ "integrity": "sha512-u2qaMkupSdhJibxvkLh3r/y36IARvnYNTLXWvOKxcQ0G/BPUB4+yF5o/yf47vv9zUV5WZv4mrdsKDt/pZDYeDg==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.782.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.782.0.tgz", - "integrity": "sha512-ZnpEvhbffaJI+Uq7HH1mVO6kUgR4uiAIIxaLa30Pqb1/oz0h4abT6W7bFOApIE5X9SDOuycoaDBEr9Ogm6ZLzQ==" + "version": "1.830.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.830.0.tgz", + "integrity": "sha512-+P4mIyp3lXkgjsr3/20Oc31ank8orQDd4K/v8TnjTQnCyxyy+VB9rynLESnkH6OlcEVSk2qf9pDM9c367E8DoQ==" }, "node_modules/windmill-parser-wasm-r": { "version": "1.668.1", @@ -14704,9 +14704,9 @@ "integrity": "sha512-9EFxeRZWmfb7EyhSlcG7dzTTKETPRYAvpRlxxLkhhtI5I219wFgI7kwrMpz4stXHJj/aqBknVv66NQHRstSJmw==" }, "node_modules/windmill-parser-wasm-wac": { - "version": "1.668.6", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-wac/-/windmill-parser-wasm-wac-1.668.6.tgz", - "integrity": "sha512-/ovcLWlIO+TQMrnWwcWoXquJI6hTZ+Zpo4POhV8z6e+pb4cVModGysASI9mERn2e9uj8/xSRmYqk54TLG8oUbQ==" + "version": "1.830.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-wac/-/windmill-parser-wasm-wac-1.830.0.tgz", + "integrity": "sha512-M67qPkzDnuMNWGrZTj9V4ajDoCQrTsQJ89UMHq308t5CtCdtiX8Z3freL+8SoRAG/dVlqQ8roK6dzncHIEM0hQ==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.770.0", diff --git a/frontend/package.json b/frontend/package.json index 259a83b5da..7522af8675 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.830.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", @@ -161,19 +161,19 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.753.0", + "windmill-parser-wasm-asset": "1.830.0", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.761.0", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.647.1", - "windmill-parser-wasm-py": "1.782.0", + "windmill-parser-wasm-py": "1.830.0", "windmill-parser-wasm-r": "1.668.1", "windmill-parser-wasm-regex": "1.764.0", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.647.1", "windmill-parser-wasm-ts": "1.695.0", - "windmill-parser-wasm-wac": "1.668.6", + "windmill-parser-wasm-wac": "1.830.0", "windmill-parser-wasm-yaml": "1.770.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "1.8.2", diff --git a/frontend/scripts/fix-svelte-module-imports.js b/frontend/scripts/fix-svelte-module-imports.js index 9cec6265a0..fe4fde4959 100644 --- a/frontend/scripts/fix-svelte-module-imports.js +++ b/frontend/scripts/fix-svelte-module-imports.js @@ -16,9 +16,22 @@ // tarball. Copy the registry into the package and rewrite those imports // to point at it (mirrors how package-system-prompts.js bundles // `$system_prompts`). +// +// 3. Chat SDK: the `windmill-chat` alias (svelte.config.js) points at +// `chat-sdk/src/index.ts`, which escapes the package root the same way. +// Compile that entry point into the package and rewrite those imports. +// +// 4. `$lib/utils` from inside `utils/`: svelte-package turns it into `./`, +// which is the `utils/` directory (no index) rather than `utils.js`. +// Point a directory specifier with no index at its sibling module. +// +// A relative import that still escapes the package root, or an +// `import.meta.glob` left on an alias, fails the build: either one +// publishes fine and only breaks in the consumer's bundler. +import { execFileSync } from 'child_process' import { copyFileSync, existsSync, readdirSync, readFileSync, statSync, writeFileSync } from 'fs' import { fileURLToPath } from 'url' -import { dirname, join, relative, resolve } from 'path' +import { dirname, join, relative, resolve, sep } from 'path' const __filename = fileURLToPath(import.meta.url) const __dirname = dirname(__filename) @@ -29,8 +42,37 @@ const oauthConnectDest = join(packageDir, 'oauth_connect.json') copyFileSync(oauthConnectSrc, oauthConnectDest) -const IMPORT_SPECIFIER_RE = /(\bfrom\s*|\bimport\s*\(?\s*)(['"])(\.\.?\/[^'"]*\.(?:svelte|json))\2/g +const chatSdkSrc = resolve(__dirname, '..', '..', 'chat-sdk', 'src', 'index.ts') +const chatSdkDest = join(packageDir, 'chat-sdk') + +// File arguments make tsc ignore every tsconfig and emit only what the entry +// point reaches, which leaves out the React and AI SDK adapters. +execFileSync( + 'npx', + [ + 'tsc', + chatSdkSrc, + '--outDir', + chatSdkDest, + '--rootDir', + dirname(chatSdkSrc), + '--declaration', + '--target', + 'ES2021', + '--module', + 'ESNext', + '--moduleResolution', + 'bundler', + '--skipLibCheck' + ], + { stdio: 'inherit', cwd: resolve(__dirname, '..') } +) + +const IMPORT_SPECIFIER_RE = /(\bfrom\s*|\bimport\s*\(?\s*)(['"])(\.\.?(?:\/[^'"]*)?)\2/g const OAUTH_CONNECT_RE = /^(\.\.\/)+backend\/oauth_connect\.json$/ +const CHAT_SDK_RE = /^(\.\.\/)+chat-sdk\/src\/index\.ts$/ +const DIRECTORY_RE = /(^|\/)\.\.?$|\/$/ +const ALIASED_GLOB_RE = /import\.meta\.glob(?:<[^>]*>)?\(\s*\[?\s*['"]\$/ function* walk(dir) { for (const entry of readdirSync(dir)) { @@ -50,9 +92,13 @@ function relativeSpecifier(fromFile, toFile) { let rewrittenFiles = 0 let rewrittenImports = 0 +const unresolvable = [] for (const file of walk(packageDir)) { const content = readFileSync(file, 'utf-8') + if (ALIASED_GLOB_RE.test(content)) { + unresolvable.push(`${relative(packageDir, file)}: import.meta.glob on an alias`) + } let changed = false const updated = content.replace(IMPORT_SPECIFIER_RE, (match, prefix, quote, specifier) => { if (OAUTH_CONNECT_RE.test(specifier)) { @@ -60,10 +106,27 @@ for (const file of walk(packageDir)) { rewrittenImports++ return `${prefix}${quote}${relativeSpecifier(file, oauthConnectDest)}${quote}` } + if (CHAT_SDK_RE.test(specifier)) { + changed = true + rewrittenImports++ + return `${prefix}${quote}${relativeSpecifier(file, join(chatSdkDest, 'index.js'))}${quote}` + } + const target = resolve(dirname(file), specifier) + if (target !== packageDir && !target.startsWith(packageDir + sep)) { + unresolvable.push(`${relative(packageDir, file)}: ${specifier}`) + return match + } + if (DIRECTORY_RE.test(specifier)) { + if (!existsSync(join(target, 'index.js')) && existsSync(target + '.js')) { + changed = true + rewrittenImports++ + return `${prefix}${quote}${relativeSpecifier(file, target + '.js')}${quote}` + } + return match + } if (!specifier.endsWith('.svelte')) { return match } - const target = resolve(dirname(file), specifier) // A real `.svelte` component file: leave the import alone. if (existsSync(target)) { return match @@ -82,6 +145,14 @@ for (const file of walk(packageDir)) { } } +if (unresolvable.length > 0) { + console.error( + `fix-svelte-module-imports: ${unresolvable.length} import(s) a consumer cannot resolve:\n` + + unresolvable.join('\n') + ) + process.exit(1) +} + console.log( `fix-svelte-module-imports: rewrote ${rewrittenImports} import(s) in ${rewrittenFiles} file(s)` ) diff --git a/frontend/src/app.html b/frontend/src/app.html index aad967fd91..3c9e232e0f 100644 --- a/frontend/src/app.html +++ b/frontend/src/app.html @@ -12,6 +12,17 @@ fill: #3b82f6; } + /* Public app pages render their shell before app.css loads, if at all. */ + html[data-wm-public-app] #svelte-global-loader { + display: none; + } + html[data-wm-public-app] body { + margin: 0; + } + html[data-wm-public-app='dark'] body { + background-color: #3b4252; + } + /* Background color for dark mode */ @media (prefers-color-scheme: dark) { #mainbg { @@ -123,6 +134,64 @@ } })() + %sveltekit.head% diff --git a/frontend/src/lib/apiBaseUrl.svelte.ts b/frontend/src/lib/apiBaseUrl.svelte.ts new file mode 100644 index 0000000000..242a6aa9e0 --- /dev/null +++ b/frontend/src/lib/apiBaseUrl.svelte.ts @@ -0,0 +1,21 @@ +import { fromStore } from 'svelte/store' +import { instanceUi } from '$lib/instanceUi' + +const ui = fromStore(instanceUi) + +/** + * Base of every URL shown for an external client to call (webhooks, HTTP routes, push + * trigger endpoints, the CLI remote): the `api_base_url` instance setting, else the + * origin being browsed. Append `/api/...` to it. + * + * Not for URLs the editor itself calls or opens (captures, previews, OAuth popups, links + * to pages): those must stay on the browsing origin, where the session cookie lives. + * + * Reactive: the setting lands after page load, so read it inside a `$derived` or the + * template rather than caching the result. + */ +export function apiBaseUrl(): string { + const setting = ui.current?.api_base_url + const configured = typeof setting === 'string' ? setting.trim() : '' + return configured || window.location.origin +} diff --git a/frontend/src/lib/base.ts b/frontend/src/lib/base.ts deleted file mode 100644 index 0e8f2d71f6..0000000000 --- a/frontend/src/lib/base.ts +++ /dev/null @@ -1 +0,0 @@ -export let base: string = import.meta.env.VITE_BASE_URL ?? '' diff --git a/frontend/src/lib/components/AgentTrace.svelte b/frontend/src/lib/components/AgentTrace.svelte index 7f2a90e621..3d671f46c8 100644 --- a/frontend/src/lib/components/AgentTrace.svelte +++ b/frontend/src/lib/components/AgentTrace.svelte @@ -1,7 +1,6 @@ + +{@render icon?.()} +{label} diff --git a/frontend/src/lib/components/BreadcrumbSegment.svelte b/frontend/src/lib/components/BreadcrumbSegment.svelte index 8a266e4fc4..4d55675f26 100644 --- a/frontend/src/lib/components/BreadcrumbSegment.svelte +++ b/frontend/src/lib/components/BreadcrumbSegment.svelte @@ -6,7 +6,7 @@ state — switching popovers relies on melt-ui's `closeOnOtherPopoverOpen` to close siblings. -->
import { ScriptService, type CiTestResult } from '$lib/gen' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' - import { base } from '$lib/base' import { CircleCheck, CircleX, Loader2, FlaskConical } from 'lucide-svelte' import { resource } from 'runed' @@ -57,14 +56,14 @@
{/if} {test.test_script_path} {#if test.job_id} view run diff --git a/frontend/src/lib/components/CliHelpBox.svelte b/frontend/src/lib/components/CliHelpBox.svelte index 0bee33825c..e91b039328 100644 --- a/frontend/src/lib/components/CliHelpBox.svelte +++ b/frontend/src/lib/components/CliHelpBox.svelte @@ -2,10 +2,11 @@ import ClipboardPanel from './details/ClipboardPanel.svelte' import Section from './Section.svelte' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' const operatingWorkspace = useOperatingWorkspace() - let url = $derived(`${window.location.protocol}//${window.location.hostname}/`) + let url = $derived(`${apiBaseUrl()}/`)
diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 375e73480e..18c765e13b 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -1,9 +1,9 @@ + +{@render children()} diff --git a/frontend/src/lib/components/CronBuilder.svelte b/frontend/src/lib/components/CronBuilder.svelte index bcf8508c12..3bceabb417 100644 --- a/frontend/src/lib/components/CronBuilder.svelte +++ b/frontend/src/lib/components/CronBuilder.svelte @@ -15,7 +15,7 @@ contentClasses="p-4" > {#snippet trigger()} - {/snippet} diff --git a/frontend/src/lib/components/DBManager.svelte b/frontend/src/lib/components/DBManager.svelte index 5204a876b9..5b34c66bc5 100644 --- a/frontend/src/lib/components/DBManager.svelte +++ b/frontend/src/lib/components/DBManager.svelte @@ -45,7 +45,12 @@ import { favoriteManager } from './sidebar/FavoriteMenu.svelte' import DatatableRoleBadge from './DatatableRoleBadge.svelte' import type { AclTarget, Asset, DataTableTables } from '$lib/gen' - import { ADMIN_DATATABLE_ROLE, datatableNameTakesRole, type DatatableRowAction } from './dbTypes' + import { + ADMIN_DATATABLE_ROLE, + datatableNameTakesRole, + namedDefaultRole, + type DatatableRowAction + } from './dbTypes' import TextInput from './text_input/TextInput.svelte' import Checkbox from './common/checkbox/Checkbox.svelte' import DbSchemaDiagram from './dbdiagram/DbSchemaDiagram.svelte' @@ -375,7 +380,8 @@ } if (!isInstanceDatatable(datatable)) return { - reason: 'Only data tables stored in the instance database have roles and grants.', + reason: + "Roles and grants are not supported on data tables backed by a Postgres resource. Store the data table on Windmill's database or on an external Postgres instance (set up by a superadmin in instance settings) to use them.", ee: false } if (needsAdmin && !canManageDatatable) @@ -416,7 +422,10 @@ if (!entry?.permissioned || !datatableNameTakesRole(datatable)) return undefined const roles = entry.usable_roles if (roles.length === 0) return { kind: 'none' } - const role = (datatable === currentDatatable ? currentRole : undefined) ?? entry.default_role + const role = + (datatable === currentDatatable ? currentRole : undefined) ?? + namedDefaultRole(entry.default_role) ?? + roles[0] if (roles.length === 1 && roles[0] === ADMIN_DATATABLE_ROLE && role === ADMIN_DATATABLE_ROLE) { return undefined } @@ -458,6 +467,11 @@ // default to open — actually be folded; a plain "expanded" set could never // close them, since the default would keep winning. // What the permissions drawer is open on, if anything: a schema, or a table. + let aclEditor = $state(undefined) + /** The open access editor lists the data table's roles, which the roles drawer can change. */ + export function refreshAcl() { + aclEditor?.refresh() + } let aclDrawer = $state<{ datatable: string | undefined; target: AclTarget } | undefined>( undefined ) @@ -1466,7 +1480,7 @@ (aclDrawer = undefined)}> (aclDrawer = undefined)} tooltip="Who owns this, and what each role may do with it." > @@ -1474,7 +1488,18 @@ {@const dt = aclDrawer.datatable ?? currentDatatable} {#if dt} {#key `${dt}~${JSON.stringify(aclDrawer.target)}`} - + onDatatableAction?.(dt, 'roles'), + blocker: accessBlocker(dt, true)?.reason + } + : undefined} + /> {/key} {/if} {/if} diff --git a/frontend/src/lib/components/DBManagerDrawer.svelte b/frontend/src/lib/components/DBManagerDrawer.svelte index bd016544b0..30fc93377d 100644 --- a/frontend/src/lib/components/DBManagerDrawer.svelte +++ b/frontend/src/lib/components/DBManagerDrawer.svelte @@ -311,6 +311,7 @@ function refreshRoles() { datatables.refetch() usableRoles.refetch() + dbManagerContent?.dbManager()?.refreshAcl() } async function handleExportSchema(explicitSource?: string) { diff --git a/frontend/src/lib/components/DatatablePicker.svelte b/frontend/src/lib/components/DatatablePicker.svelte index 53617acf36..17ad6f3361 100644 --- a/frontend/src/lib/components/DatatablePicker.svelte +++ b/frontend/src/lib/components/DatatablePicker.svelte @@ -17,6 +17,9 @@ selectInputClass?: string class?: string onClear?: () => void + /** Size the input to its content, like Path's folder picker. */ + useContentEditable?: boolean + RightIcon?: any } let { @@ -27,7 +30,9 @@ placeholder = undefined, selectInputClass = '', class: className = '', - onClear = undefined + onClear = undefined, + useContentEditable = false, + RightIcon = undefined }: Props = $props() let datatables = usePromise(() => @@ -47,6 +52,8 @@ {placeholder} inputClass={selectInputClass} {onClear} + {useContentEditable} + {RightIcon} /> {#if showSchemaExplorer && value && assetCanBeExplored({ kind: 'datatable', path: value })} diff --git a/frontend/src/lib/components/DefaultScriptsInner.svelte b/frontend/src/lib/components/DefaultScriptsInner.svelte index b062fb4c49..29136e42f7 100644 --- a/frontend/src/lib/components/DefaultScriptsInner.svelte +++ b/frontend/src/lib/components/DefaultScriptsInner.svelte @@ -16,11 +16,13 @@ let { small = false }: Props = $props() function computeLangs(defaultScripts: WorkspaceDefaultScripts | undefined): Script['language'][] { - const allLangs = Object.keys(defaultScriptLanguages) as Script['language'][] + const allLangs = (Object.keys(defaultScriptLanguages) as Script['language'][]).filter( + (l) => l != 'deno' + ) if (!defaultScripts || defaultScripts.order == undefined) return allLangs return defaultScripts.order ?.concat(allLangs.filter((l) => !defaultScripts.order?.includes(l))) - .filter((x) => x != 'nativets') as Script['language'][] + .filter((x) => x != 'nativets' && x != 'deno') as Script['language'][] } async function changePosition(i: number, up: boolean) { @@ -31,6 +33,10 @@ ;[norder[i], norder[i + 1]] = [norder[i + 1], norder[i]] } defaultScripts.update((s) => ({ ...s, order: norder })) + await persist() + } + + async function persist() { await WorkspaceService.editDefaultScripts({ workspace: $operatingWorkspace!, requestBody: $defaultScripts @@ -76,7 +82,7 @@ size="xs" color="red" checked={$defaultScripts?.hidden?.includes(lang)} - on:change={(e) => { + on:change={async (e) => { let toggled = e.detail if (toggled) { defaultScripts.update((s) => ({ @@ -89,6 +95,7 @@ hidden: (s?.hidden ?? []).filter((h) => h != lang) })) } + await persist() }} />
diff --git a/frontend/src/lib/components/DeployWorkspace.svelte b/frontend/src/lib/components/DeployWorkspace.svelte index 53ca12cf03..7913240188 100644 --- a/frontend/src/lib/components/DeployWorkspace.svelte +++ b/frontend/src/lib/components/DeployWorkspace.svelte @@ -5,7 +5,6 @@ aiAgentModuleDependencies, collectTransformRefs } from './deployDependencies' - import { base } from '$lib/base' import { enterpriseLicense, superadmin } from '$lib/stores' import { AppService, @@ -538,7 +537,7 @@ > {:else} Share this link to have another properly permissioned + >Share this link to have another properly permissioned user do the deployment @@ -580,7 +579,7 @@ connection={remoteStatus?.connection} returnTo={kind === 'trigger' ? undefined - : `${base}/deploy/${kind}/${initialPath}?workspace=${encodeURIComponent($operatingWorkspace!)}`} + : `/deploy/${kind}/${initialPath}?workspace=${encodeURIComponent($operatingWorkspace!)}`} onChange={loadDestinations} /> diff --git a/frontend/src/lib/components/DevWorkspaceBadge.svelte b/frontend/src/lib/components/DevWorkspaceBadge.svelte new file mode 100644 index 0000000000..e86e4593c8 --- /dev/null +++ b/frontend/src/lib/components/DevWorkspaceBadge.svelte @@ -0,0 +1,56 @@ + + + +{devBadgeText(label)} diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte index e148d90c22..5346cab7b0 100644 --- a/frontend/src/lib/components/DevWorkspaceSetting.svelte +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -1,13 +1,13 @@ -
+
{#if !hidePath}
@@ -162,9 +111,9 @@ {#if segments} {#each segments.dirs as dir, i (dir.fullPath)} {@const dKey = dirKey('all', dir.fullPath)} + {/each} + {/if} + {#if segments?.leaf} {@const leafKey = leafKeyFor(kind, segments.leaf.fullPath)} {@const leafParent = segments.dirs[segments.dirs.length - 1]?.fullPath} + - + {#if pathEditable} - pathPopoverOpen, setPathPopoverOpen} - > - {#snippet trigger()} -
{/if} diff --git a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte index 0adb5dc690..c624b5a148 100644 --- a/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte +++ b/frontend/src/lib/components/ErrorOrRecoveryHandler.svelte @@ -43,7 +43,6 @@ import ChannelSelector from '$lib/components/ChannelSelector.svelte' import type { Schema, SupportedLanguage } from '$lib/common' - import { base } from '$lib/base' import { enterpriseLicense } from '$lib/stores' import MsTeamsIcon from '$lib/components/icons/MSTeamsIcon.svelte' import { classNames, emptySchema, emptyString, sendUserToast, tryEvery } from '$lib/utils' @@ -70,6 +69,8 @@ } from 'lucide-svelte' import { hubPaths } from '$lib/hub' import { isCloudHosted } from '$lib/cloud' + import { untrack } from 'svelte' + import { syncSlackBotArg } from './handlerSlackArg' import SmtpConfigurationStatus from './common/smtp/SmtpConfigurationStatus.svelte' import { SettingService } from '$lib/gen' import { isSmtpSettingsValid } from './instanceSettings/SmtpSettings.svelte' @@ -317,12 +318,13 @@ loadSmtpConfiguration() }) + let builtinArgs: Record | undefined $effect(() => { - if (handlerSelected === 'slack' && isSlackHandler(handlerPath)) { - handlerExtraArgs['slack'] = '$res:f/slack_bot/bot_token' - } else { - handlerExtraArgs['slack'] = undefined - } + const isSlackBuiltin = handlerSelected === 'slack' && isSlackHandler(handlerPath) + const args = handlerExtraArgs + // A replaced args object (Discard, reload) holds the stored config, not what the built-in set. + untrack(() => syncSlackBotArg(args, isSlackBuiltin, args === builtinArgs)) + builtinArgs = isSlackBuiltin ? args : undefined }) let lastHandlerSelected: ErrorHandler | undefined = $state(undefined) @@ -543,7 +545,7 @@ {connectionTestJob.uuid} @@ -636,7 +638,7 @@ Message sent via Windmill job {connectionTestJob.uuid} @@ -694,7 +696,7 @@ report is sent without adding an entry to the instance critical alert feed. Canceled jobs are not reported. - + Configure the instance critical alert channels
diff --git a/frontend/src/lib/components/FilterSearchbar.svelte b/frontend/src/lib/components/FilterSearchbar.svelte index b4bc4471a2..0a65701b06 100644 --- a/frontend/src/lib/components/FilterSearchbar.svelte +++ b/frontend/src/lib/components/FilterSearchbar.svelte @@ -303,7 +303,12 @@ {#if showBanner} - -
-
- -
-
- -
- - {isDevWorkspace - ? `${devLabelWord(currentWorkspaceData?.dev_workspace_label)} workspace of` - : 'Fork of'} - {parentWorkspaceData?.name}{#if parentWorkspaceData?.name !== parentWorkspaceId} - ({parentWorkspaceId}){/if} - -
- - {#if loading} - Checking for changes... - {:else if error} - - {error} - - {:else if comparison} -
- {#if comparison.summary.total_diffs > 0} - - {forkAheadBehindMessage(changesAhead, changesBehind)} - {parentWorkspaceId} over {comparison - .summary.total_diffs} items - - - - - {#if ciTestTotal > 0} - - - {#if ciTestFailing > 0} -
- - CI: {ciTestFailing} failing -
- {:else if ciTestRunning > 0} -
- - CI: {ciTestRunning} running -
- {:else} -
- - CI: {ciTestPassing} passing -
- {/if} - {/if} - - {#if comparison.summary.conflicts > 0} - - -
- - {comparison.summary.conflicts} conflict{comparison.summary.conflicts !== 1 - ? 's' - : ''} -
- {/if} - {:else if comparison.skipped_comparison} - - This {currentNoun} was created before the addition of certain windmill features, and - therefore the changes with its parent workspace cannot be displayed. - {:else if showDraftsOnly} - - This workspace has {draftCount} draft{draftCount !== 1 ? 's' : ''} - - {:else} - Everything is up to date - {/if} -
- {/if} -
- -
- -
-
-
-
+ + {/if} + +{#snippet forkAction()} + {@const total = comparison?.summary.total_diffs ?? 0} + {@const conflicts = comparison?.summary.conflicts ?? 0} + + {#if loading} + Checking for changes… + {:else if error} + Comparison failed + {:else if showDraftsOnly} + + {draftCount} draft{draftCount !== 1 ? 's' : ''} + + {:else if total > 0} + + + {#if changesAhead > 0} + openCompareDir('deploy_to')} + title="{changesAhead} ahead of {parentWorkspaceId} — review what this {currentNoun} has to give" + > + + {changesAhead}{compactCounts ? '' : ' ahead'} + + {/if} + {#if changesBehind > 0} + openCompareDir('update')} + title="{changesBehind} behind {parentWorkspaceId} — review what this {currentNoun} has yet to take" + > + + {changesBehind}{compactCounts ? '' : ' behind'} + + {/if} + {#if conflicts > 0} + + openCompareDir('deploy_to')} + title="{conflicts} conflicting item{conflicts !== 1 ? 's' : ''} — review them" + > + + {conflicts}{compactCounts ? '' : ` conflict${conflicts !== 1 ? 's' : ''}`} + + {/if} + + {#if draftCount > 0} + + + {draftCount}{compactCounts ? '' : ` draft${draftCount !== 1 ? 's' : ''}`} + + {/if} + {/if} + {#if showDraftsOnly} + + {:else} + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/GitRepoViewer.svelte b/frontend/src/lib/components/GitRepoViewer.svelte index 53ddea2a96..d00449f168 100644 --- a/frontend/src/lib/components/GitRepoViewer.svelte +++ b/frontend/src/lib/components/GitRepoViewer.svelte @@ -3,7 +3,6 @@ import { untrack } from 'svelte' import { Alert, Button } from './common' import S3FilePickerInner from './S3FilePickerInner.svelte' - import { base } from '$lib/base' import { Loader, Loader2, ChevronDown, ChevronRight, ExternalLink } from 'lucide-svelte' import { hubPaths } from '$lib/hub' import { sleep } from '$lib/utils' @@ -237,7 +236,7 @@ {#if runningJobId} @@ -272,7 +271,7 @@ {#if runningJobId} diff --git a/frontend/src/lib/components/GroupEditor.svelte b/frontend/src/lib/components/GroupEditor.svelte index f8d0445840..b4229cf9f6 100644 --- a/frontend/src/lib/components/GroupEditor.svelte +++ b/frontend/src/lib/components/GroupEditor.svelte @@ -15,7 +15,7 @@ import Row from './table/Row.svelte' import Cell from './table/Cell.svelte' import { sendUserToast } from '$lib/toast' - import { canWrite } from '$lib/utils' + import { useEditRights } from '$lib/operatorWriteRights' import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import Label from './Label.svelte' @@ -42,6 +42,7 @@ const operatingWorkspace = useOperatingWorkspace() const operatingUser = useOperatingUser() const actingUser = $derived(operatingUser.current) + const editRights = useEditRights() const ROLE_TOOLTIPS = { member: @@ -92,7 +93,7 @@ // Derived for the same reason as the trigger editors: the acting user's role can land after // the group does, and a snapshot taken first would stay read-only. const can_write = $derived( - writeVerdict ?? (groupPerms !== undefined && canWrite(name, groupPerms, actingUser)) + writeVerdict ?? (groupPerms !== undefined && editRights.canEditItem('group', name, groupPerms)) ) let group: Group | undefined let instance_group: InstanceGroup | undefined = $state() diff --git a/frontend/src/lib/components/HeaviestScriptsDrawer.svelte b/frontend/src/lib/components/HeaviestScriptsDrawer.svelte new file mode 100644 index 0000000000..a4591924c9 --- /dev/null +++ b/frontend/src/lib/components/HeaviestScriptsDrawer.svelte @@ -0,0 +1,201 @@ + + + (opened = false)}> + +
+ {#snippet action()} + + {#snippet children({ item })} + {#each Object.keys(WINDOWS) as key (key)} + + {/each} + {/snippet} + + {/snippet} + +
+
+ Sort by + orderBy, + (v) => { + orderBy = v + logFeatureUsage('worker_resource_stats', 'sort', { key: v }) + } + } + noWFull + > + {#snippet children({ item })} + {#each Object.entries(ORDERS) as [value, label] (value)} + + {/each} + {/snippet} + +
+
+
+ Script + Workspace + Worker group + Jobs + Total runtime + Avg runtime + Max memory + Avg memory + CPU time + Avg CPUs + + + + {#each stats.current as s (s.workspace_id + '\0' + s.runnable_path + '\0' + s.worker_group)} + + + {#if s.runnable_path === ADHOC_PATH} + Previews and other ad-hoc runs + {:else} + {s.runnable_path} + {/if} + + {s.workspace_id} + {s.worker_group} + {s.job_count} + + {duration(s.total_duration_ms)} + + + + {s.job_count > 0 ? duration(s.total_duration_ms / s.job_count) : '-'} + + {#if s.memory_sample_count > 0} + + {formatMemory(s.max_memory_peak)} + + + {formatMemory(Math.round(s.sum_memory_peak / s.memory_sample_count))} + + {:else} + - + - + {/if} + + {duration(s.total_cpu_ms)} + + + {cores(s.total_cpu_ms, s.total_duration_ms)} + + + {/each} + + + {/if} + + + diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 1476b9ad87..a1a0f214d3 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -29,6 +29,7 @@ import SettingsPageHeader from './settings/SettingsPageHeader.svelte' import WorkspaceRegistries from './instanceSettings/WorkspaceRegistries.svelte' import DbHealth from './instanceSettings/DbHealth.svelte' + import OidcSigningStatus from './instanceSettings/OidcSigningStatus.svelte' interface Props { tab?: string @@ -1186,6 +1187,15 @@ description="On-demand database diagnostics. Analyze table sizes, job retention, connection pool health, vacuum status, and more." /> + {:else if category == 'OIDC signing'} + + {#if $enterpriseLicense} + + {/if} {:else if category == 'Auth/OAuth/SAML'} BASE_URL="{window.location.origin}" + >BASE_URL="{apiBaseUrl()}" WM_TOKEN="{''}" diff --git a/frontend/src/lib/components/LogViewer.svelte b/frontend/src/lib/components/LogViewer.svelte index be4ecb5b36..f15b87f9ef 100644 --- a/frontend/src/lib/components/LogViewer.svelte +++ b/frontend/src/lib/components/LogViewer.svelte @@ -14,7 +14,6 @@ import { ClipboardCopy, Download, Expand, Loader2, Timer, Cpu } from 'lucide-svelte' import { Button, Drawer, DrawerContent } from './common' import { copyToClipboard } from '$lib/utils' - import { base } from '$lib/base' import { withExternalDomain } from '$lib/externalDomain' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' import { appendViewToken } from '$lib/viewToken' @@ -253,7 +252,7 @@ } }) let logsApiPath = $derived(appendViewToken(`/w/${$operatingWorkspace}/jobs_u/get_logs/${jobId}`)) - let downloadHref = $derived(withExternalDomain(`${base}/api${logsApiPath}`)) + let downloadHref = $derived(withExternalDomain(`/api${logsApiPath}`)) let downloadName = $derived(`windmill_logs_${jobId}.txt`) let truncatedContent = $derived( truncateContent(effectiveContent, loadedFromObjectStore, LOG_LIMIT) diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 76f4a2b995..1199d01079 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -30,7 +30,6 @@ import { OauthService, UserService, WorkspaceService } from '$lib/gen' import { usersWorkspaceStore, workspaceStore, userStore } from '$lib/stores' import { emptyString, escapeHtml, parseQueryParams } from '$lib/utils' - import { base } from '$lib/base' import { getUserExt } from '$lib/user' import { sendUserToast } from '$lib/toast' import { isCloudHosted } from '$lib/cloud' @@ -354,7 +353,7 @@ // See (root)/+layout.svelte for why /projects/import skips the picker. } else if ( resolvedRd?.startsWith('/user/workspaces') || - resolvedRd?.startsWith(`${base}/projects/import`) + resolvedRd?.startsWith('/projects/import') ) { goto(resolvedRd) } else if (resolvedRd == '/#user-settings') { @@ -575,7 +574,7 @@ if (automatic) params.set('auto', 'true') if (guestApp) params.set('guest_app', guestApp) const query = params.size > 0 ? '?' + params.toString() : '' - let url = base + '/api/oauth/login/' + provider + query + let url = '/api/oauth/login/' + provider + query console.log('storeRedirect', popup, url) if (popup) { @@ -856,7 +855,7 @@ {#if smtpConfigured}
Forgot password? diff --git a/frontend/src/lib/components/MemoryFootprintViewer.svelte b/frontend/src/lib/components/MemoryFootprintViewer.svelte index 9547861bc3..14ad1a2b06 100644 --- a/frontend/src/lib/components/MemoryFootprintViewer.svelte +++ b/frontend/src/lib/components/MemoryFootprintViewer.svelte @@ -3,7 +3,7 @@ import { type MetricDataPoint, MetricsService } from '$lib/gen' import { displayTime } from '$lib/utils' - import { workspaceStore } from '$lib/stores' + import { useOperatingWorkspace } from './operatingWorkspace.svelte' import { CategoryScale, Chart as ChartJS, @@ -26,6 +26,8 @@ let { jobId, jobUpdateLastFetch }: Props = $props(); + const operatingWorkspace = useOperatingWorkspace() + let jobMetricsLastFetch: Date | undefined = undefined let jobMemoryStats: MetricDataPoint[] | undefined = $state(undefined) @@ -38,7 +40,7 @@ async function loadMetricsData() { try { let jobStatsPromise = MetricsService.getJobMetrics({ - workspace: $workspaceStore!, + workspace: $operatingWorkspace!, id: jobId, requestBody: { from_timestamp: jobMetricsLastFetch?.toISOString(), diff --git a/frontend/src/lib/components/NavBreadcrumb.svelte b/frontend/src/lib/components/NavBreadcrumb.svelte new file mode 100644 index 0000000000..b053591fc0 --- /dev/null +++ b/frontend/src/lib/components/NavBreadcrumb.svelte @@ -0,0 +1,434 @@ + + + +{#snippet workspaceDisc()} + + + + + + + +{/snippet} + +{#snippet itemKindIcon()} + + {#if item?.kind}{/if} +{/snippet} + +{#snippet sectionKindIcon()} + {#if section?.kind}{/if} +{/snippet} + +{#snippet folderIcon()} + +{/snippet} + +{#snippet userIcon()} + +{/snippet} + +{#snippet envBadgeMark()} + {#if envBadge === 'dev'} + + {:else if envBadge === 'prod'} + prod + {/if} +{/snippet} + +{#snippet slash(extra = '')} + + +{/snippet} + + diff --git a/frontend/src/lib/components/NoDirectDeployAlert.svelte b/frontend/src/lib/components/NoDirectDeployAlert.svelte index d1017f7347..a3a249caf5 100644 --- a/frontend/src/lib/components/NoDirectDeployAlert.svelte +++ b/frontend/src/lib/components/NoDirectDeployAlert.svelte @@ -16,7 +16,8 @@ import { findCanonicalDevWorkspace } from '$lib/utils/workspaceHierarchy' import { devLabelKey, devLabelNoun } from '$lib/utils/devWorkspaceLabel' import { switchWorkspace } from '$lib/storeUtils' - import { Badge, Button } from './common' + import { Button, ButtonType } from './common' + import { twMerge } from 'tailwind-merge' import Popover from './meltComponents/Popover.svelte' import Toggle from './Toggle.svelte' import { GitFork, Lock, ShieldOff } from 'lucide-svelte' @@ -83,28 +84,35 @@ ? `Edits in ${devLabelKey(canonicalDev.dev_workspace_label)}` : 'Edits restricted' ) + + // The Popover renders its trigger as a
diff --git a/frontend/src/lib/components/ResourceEditor.svelte b/frontend/src/lib/components/ResourceEditor.svelte index 9847454aca..bd9d51638d 100644 --- a/frontend/src/lib/components/ResourceEditor.svelte +++ b/frontend/src/lib/components/ResourceEditor.svelte @@ -7,7 +7,8 @@ type Resource, type ResourceType } from '$lib/gen' - import { canWrite } from '$lib/utils' + import { canEditItem } from '$lib/editRights' + import { userWorkspaces } from '$lib/stores' import { createEventDispatcher, onDestroy, untrack } from 'svelte' import { sendUserToast } from '$lib/toast' import { clearJsonSchemaResourceCache } from './schema/jsonSchemaResource.svelte' @@ -261,8 +262,16 @@ if (!initialPath || !selected) return true const r = fetchedResources[selected] if (!r || !acting.resolved(selected)) return undefined - return canWrite(current?.path ?? initialPath, r.extra_perms ?? {}, acting.in(selected)) + return canEditResourceIn(selected, current?.path ?? initialPath, r.extra_perms) }) + function canEditResourceIn( + ws: string, + path: string, + extraPerms: Record | undefined + ): boolean { + const settings = $userWorkspaces.find((w) => w.id === ws)?.operator_settings + return canEditItem('resource', path, extraPerms, acting.in(ws), settings) + } const dirtyWorkspaces = $derived( Object.keys(states).filter((ws) => !draftValuesEqual(states[ws].draft, initialStates[ws])) @@ -423,9 +432,7 @@ const dirtyCanWrite = $derived( dirtyWorkspaces.every((ws) => { const r = fetchedResources[ws] - return ( - !r || canWrite(states[ws]?.draft?.path ?? initialPath, r.extra_perms ?? {}, acting.in(ws)) - ) + return !r || canEditResourceIn(ws, states[ws]?.draft?.path ?? initialPath, r.extra_perms) }) ) diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte index ff32cbdbd7..3cc3bb2ffd 100644 --- a/frontend/src/lib/components/RunsPage.svelte +++ b/frontend/src/lib/components/RunsPage.svelte @@ -11,23 +11,13 @@ import { sendUserToast } from '$lib/toast' import { displayDate } from '$lib/utils' + import { userWorkspaces, superadmin, devopsRole, enterpriseLicense } from '$lib/stores' + import { useHostedPage } from '$lib/components/hostedPage' import { - userStore, - workspaceStore, - userWorkspaces, - superadmin, - devopsRole, - enterpriseLicense - } from '$lib/stores' - import { - Button, - ButtonType, - Drawer, - DrawerContent, - Skeleton, - Tab, - Tabs - } from '$lib/components/common' + useOperatingUser, + useOperatingWorkspace + } from '$lib/components/operatingWorkspace.svelte' + import { Button, Drawer, DrawerContent, Skeleton, Tab, Tabs } from '$lib/components/common' import TextInput from '$lib/components/text_input/TextInput.svelte' import RunChart from '$lib/components/RunChart.svelte' @@ -44,7 +34,9 @@ import ConfirmationModal from '$lib/components/common/confirmationModal/ConfirmationModal.svelte' import RunsQueue from '$lib/components/runs/RunsQueue.svelte' import OpenInSessionButton from '$lib/components/sessions/OpenInSessionButton.svelte' - import { pageHref, RUNS_PATH } from '$lib/components/sessions/previewPaths' + import PageHeaderContent from '$lib/components/PageHeaderContent.svelte' + import { pageHeader, PHONE_BAR } from '$lib/components/pageHeaderRegistry.svelte' + import { RUNS_PATH } from '$lib/components/sessions/previewPaths' import { twMerge } from 'tailwind-merge' import { computeJobKinds, useJobsLoader } from '$lib/components/runs/useJobsLoader.svelte' import ConcurrentJobsChart from '$lib/components/ConcurrentJobsChart.svelte' @@ -87,6 +79,13 @@ let { initialPath }: Props = $props() + // Under a host — an AI session's preview tab — the document URL, the page header band and + // the navigation workspace all belong to the host: the filters live where it keeps them, + // the filter row renders in the page, and requests go to the workspace it names. + const hosted = useHostedPage() + const operatingWorkspace = useOperatingWorkspace() + const operatingUser = useOperatingUser() + let paths: string[] = $state([]) let usernames: string[] = $state([]) let folders: string[] = $state([]) @@ -97,7 +96,7 @@ folders, jobTriggerKinds, isSuperAdminOrDevops: !!$superadmin || !!$devopsRole, - isAdminsWorkspace: $workspaceStore === 'admins' + isAdminsWorkspace: $operatingWorkspace === 'admins' }) ) let perPage = useLocalStorageValue('runs_per_page', 1000, 'number') @@ -121,6 +120,8 @@ // what was seeded at mount is gone. Re-apply it. Editing a filter writes with `replaceState`, // which never reaches `page.url`, so a filter the user clears stays cleared. $effect(() => { + // A host remounts the page when it re-points it, and its document URL is not this page's. + if (hosted) return page.url.href untrack(() => filters.seed(filterSeeds())) }) @@ -177,7 +178,9 @@ let manualTimeframe = $derived(timeframe.type === 'manual' ? timeframe : undefined) let graph: 'RunChart' | 'ConcurrencyChart' = $state( - typeOfChart(page.url.searchParams.get('graph')) + typeOfChart( + (hosted ? new URLSearchParams(hosted.search) : page.url.searchParams).get('graph') + ) ) let innerWidth = $state(window.innerWidth) let jobsLoader = useJobsLoader(() => ({ @@ -189,7 +192,7 @@ resultError, perPage: perPage.val, lookback: graph === 'RunChart' ? 0 : lookback, - currentWorkspace: $workspaceStore ?? '' + currentWorkspace: $operatingWorkspace ?? '' })) let batchProgress = $derived(jobsLoader.batchProgress) let currentBatchSize = $derived(jobsLoader.currentBatchSize) @@ -217,18 +220,18 @@ } async function loadUsernames(): Promise { - usernames = await UserService.listUsernames({ workspace: $workspaceStore! }) + usernames = await UserService.listUsernames({ workspace: $operatingWorkspace! }) } async function loadFolders(): Promise { folders = await FolderService.listFolders({ - workspace: $workspaceStore! + workspace: $operatingWorkspace! }).then((x) => x.map((y) => y.name)) } async function loadPaths() { - const npaths_scripts = await ScriptService.listScriptPaths({ workspace: $workspaceStore ?? '' }) - const npaths_flows = await FlowService.listFlowPaths({ workspace: $workspaceStore ?? '' }) + const npaths_scripts = await ScriptService.listScriptPaths({ workspace: $operatingWorkspace ?? '' }) + const npaths_flows = await FlowService.listFlowPaths({ workspace: $operatingWorkspace ?? '' }) paths = npaths_scripts.concat(npaths_flows).sort() } @@ -271,7 +274,7 @@ const resultFilter = filters.val.result && JSON.parse(filters.val.result) const { minTs, maxTs } = timeframe.computeMinMax() return { - workspace: $workspaceStore ?? '', + workspace: $operatingWorkspace ?? '', startedBefore: maxTs ?? undefined, startedAfter: minTs ?? undefined, schedulePath: filters.val.schedule_path ?? undefined, @@ -329,7 +332,7 @@ } async function cancelJobs(uuidsToCancel: string[], forceCancel: boolean = false) { const uuids = await JobService.cancelSelection({ - workspace: $workspaceStore ?? '', + workspace: $operatingWorkspace ?? '', requestBody: uuidsToCancel, forceCancel: forceCancel, allWorkspaces: filters.val.all_workspaces || undefined @@ -356,7 +359,7 @@ for (const j of jobs ?? []) if (j.workspace_id) workspaceById.set(j.id, j.workspace_id) const byWorkspace = new Map() for (const id of jobIds) { - const ws = workspaceById.get(id) ?? $workspaceStore ?? '' + const ws = workspaceById.get(id) ?? $operatingWorkspace ?? '' const bucket = byWorkspace.get(ws) if (bucket) bucket.push(id) else byWorkspace.set(ws, [id]) @@ -434,7 +437,7 @@ } async function reRunJobs(jobIdsToReRun: string[], batchReRunOptions: BatchReRunOptions) { - if (!$workspaceStore) return + if (!$operatingWorkspace) return if (askingForConfirmation) { askingForConfirmation.loading = true @@ -448,7 +451,7 @@ // workaround because EventSource does not support POST requests // https://medium.com/@david.richards.tech/sse-server-sent-events-using-a-post-request-without-eventsource-1c0bd6f14425 - const response = await fetch(`${OpenAPI.BASE}/w/${$workspaceStore}/jobs/run/batch_rerun_jobs`, { + const response = await fetch(`${OpenAPI.BASE}/w/${$operatingWorkspace}/jobs/run/batch_rerun_jobs`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) @@ -561,7 +564,7 @@ } $effect(() => { - if ($workspaceStore) { + if ($operatingWorkspace) { untrack(() => { loadUsernames() loadFolders() @@ -609,8 +612,248 @@ openedRunId = undefined } let resolutionNote = $state('') + const phone = $derived(!hosted && pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR) +{#snippet runsHint()} + + All past and schedule executions of scripts and flows, including previews. You only see your own + runs or runs of groups you belong to unless you are an admin. + +{/snippet} + +{#snippet headerActions()} + +
+ +
+ { + jobsFilter('waiting') + }} + onJobsSuspended={() => { + jobsFilter('suspended') + }} + small + /> +
+ + + + + + + + jobsLoader?.loadJobs(true)} + loading={jobsLoader?.loading} + items={runsTimeframes} + bind:value={_timeframe.val} + /> + +
+ RUNS_PATH + window.location.search, + workspaceId: $operatingWorkspace ?? undefined + }} + btnProps={{ unifiedSize: 'sm' }} + /> +
+
+{/snippet} + + { - reset() + if (!hosted) reset() }} /> -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.runs} +{#if operatingUser.current?.operator && $operatingWorkspace && !$userWorkspaces.find((_) => _.id === $operatingWorkspace)?.operator_settings?.runs} {:else} -
- -
-
-

- Runs -

- - - All past and schedule executions of scripts and flows, including previews. You only see - your own runs or runs of groups you belong to unless you are an admin. - - - - { - jobsFilter('waiting') - }} - onJobsSuspended={() => { - jobsFilter('suspended') - }} - small={innerWidth < smallScreenWidth} - /> +
+ + {#if hosted} +
+ {@render headerActions()}
- - - - - - jobsLoader?.loadJobs(true)} - loading={jobsLoader?.loading} - items={runsTimeframes} - bind:value={_timeframe.val} - /> - -
- - - -
- pageHref(RUNS_PATH) + window.location.search, - workspaceId: $workspaceStore ?? undefined - }} - btnProps={{ unifiedSize: 'md' }} - /> -
-
-
+ {:else} + + {/if}
@@ -1093,7 +1122,7 @@ batchRerunOptionsIsOpen = true } }, - ...(!$userStore?.operator + ...(!operatingUser.current?.operator ? [ { // Operators are rejected by the endpoint, so offering it would only 403. diff --git a/frontend/src/lib/components/S3FilePicker.svelte b/frontend/src/lib/components/S3FilePicker.svelte index ce1af47118..a5e4849c57 100644 --- a/frontend/src/lib/components/S3FilePicker.svelte +++ b/frontend/src/lib/components/S3FilePicker.svelte @@ -105,6 +105,7 @@ size="1200px" > { s3FilePickerInner?.exit?.() diff --git a/frontend/src/lib/components/S3FilePickerInner.svelte b/frontend/src/lib/components/S3FilePickerInner.svelte index 1bf1b4358a..89c23e84ae 100644 --- a/frontend/src/lib/components/S3FilePickerInner.svelte +++ b/frontend/src/lib/components/S3FilePickerInner.svelte @@ -8,8 +8,35 @@ Loader2, Download, Trash, - MoveRight + MoveRight, + Ellipsis, + FileText, + FileCode, + FileJson, + FileSpreadsheet, + FileImage, + FileArchive, + FilePlay, + FileAudio, + Table2 } from 'lucide-svelte' + import { Pane, Splitpanes } from 'svelte-splitpanes' + import TextInput from './text_input/TextInput.svelte' + import Select from './select/Select.svelte' + import Toggle from './Toggle.svelte' + import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' + import type { IDbTableOps } from './dbOps' + import { resource } from 'runed' + import { staticDbTableOps } from './staticDbTableOps' + import { + fetchJsonTableRows, + isJsonFile, + JSON_TABLE_MAX_BYTES, + s3TableOps, + serverTableFormat, + type S3TableSource + } from './s3TableOps' import { CancelablePromise, HelpersService, @@ -28,21 +55,19 @@ type MoveS3FileData, type MoveS3FileResponse } from '$lib/gen' - import { base } from '$lib/base' import { displayDate, displaySize, emptyString, parseS3Object, sendUserToast, - type S3Object + type S3Object, + type IconType } from '$lib/utils' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' - import { Alert, Button } from './common' - import Section from './Section.svelte' + import { Alert, Button, CopyButton } from './common' import { createEventDispatcher, untrack, type Snippet } from 'svelte' import VirtualList from '@tutorlatin/svelte-tiny-virtual-list' - import TableSimple from './TableSimple.svelte' import ConfirmationModal from './common/confirmationModal/ConfirmationModal.svelte' import FileUploadModal from './common/fileUpload/FileUploadModal.svelte' import S3FilePreview from './S3FilePreview.svelte' @@ -144,8 +169,14 @@ let rootPath = $state(initialRootPath) let rootPathNestingLevel = $derived(1 * (rootPath.split('/').length - 1)) - let csvSeparatorChar: string = $state(',') + let csvSeparatorChar: string | undefined = $state(undefined) let csvHasHeader: boolean = $state(true) + const CSV_SEPARATORS = [ + { value: ',', label: ',' }, + { value: ';', label: ';' }, + { value: '\t', label: 'Tab' }, + { value: '|', label: '|' } + ] let dispatch = createEventDispatcher<{ close: { s3: string; storage: string | undefined } | undefined @@ -179,6 +210,68 @@ } | undefined = $state(undefined) + let viewMode: 'table' | 'text' = $state('table') + /** The selected file, when it can be shown as a table. Its own requests can't read + * files behind a custom listing (git repos), so those keep the text preview. */ + let tableSource = $derived.by(() => { + if (hideS3SpecificDetails || !fileMetadata || !ws) return undefined + const fileKey = fileMetadata.fileKey + const inBrowser = isJsonFile(fileKey) && (fileMetadata.size ?? Infinity) <= JSON_TABLE_MAX_BYTES + if (!inBrowser && !serverTableFormat(fileKey)) return undefined + return { + workspace: ws, + fileKey, + storage, + s3ResourcePath, + csvSeparator: csvSeparatorChar, + csvHasHeader, + inBrowser + } + }) + let tableKey = $derived(tableSource && JSON.stringify(tableSource)) + let tableOps = resource( + () => (viewMode === 'table' ? tableSource : undefined), + async (source, _previous, { signal }) => { + if (!source) return undefined + const key = JSON.stringify(source) + // A response for a file or options since left must not replace the current one; + // the resource drops a fetch that throws an AbortError. + const superseded = () => new DOMException('Superseded', 'AbortError') + try { + const table = source.inBrowser ? await jsonTable(source) : await s3TableOps(source) + if (signal.aborted) throw superseded() + return { key, ops: table?.ops, csvSeparator: table?.csvSeparator } + } catch (e) { + throw signal.aborted ? superseded() : e + } + } + ) + async function jsonTable( + source: S3TableSource + ): Promise<{ ops: IDbTableOps; csvSeparator?: string } | undefined> { + const rows = await fetchJsonTableRows(source) + return rows && { ops: staticDbTableOps(rows, 'duckdb', source.fileKey.split('/').pop() ?? '') } + } + /** The loaded table, once it belongs to the selected file and options. */ + let currentTable = $derived(tableOps.current?.key === tableKey ? tableOps.current : undefined) + /** A JSON file that turns out not to be an array has no table to show. */ + let showTable = $derived( + viewMode === 'table' && !!tableSource && !(currentTable && !currentTable.ops) + ) + // Each file starts on the separator the server guesses from its first bytes. + $effect(() => { + fileMetadata?.fileKey + untrack(() => { + csvSeparatorChar = undefined + detectedCsvSeparator = undefined + }) + }) + /** Kept while the file stays selected, so the text view still shows what was guessed. */ + let detectedCsvSeparator: string | undefined = $state(undefined) + $effect(() => { + if (currentTable?.csvSeparator) detectedCsvSeparator = currentTable.csvSeparator + }) + /** Identifies the metadata request that currently owns the preview pane. */ let metadataRequestId = 0 @@ -229,6 +322,25 @@ /** Per-level listing only makes sense while browsing; searching stays flat. */ let lazyMode = $derived(lazyFolders && filter.trim() === '') + /** `nestingLevel` counts two per folder depth. */ + function treeRowIndent(nestingLevel: number): string { + return `padding-left: ${12 + (nestingLevel / 2) * 16}px;` + } + + const FILE_ICONS: [RegExp, IconType][] = [ + [/\.(csv|tsv|xlsx?|parquet|ods)$/i, FileSpreadsheet], + [/\.(json|jsonl|ndjson)$/i, FileJson], + [/\.(png|jpe?g|gif|webp|svg|bmp|ico|tiff?)$/i, FileImage], + [/\.(zip|tar|gz|tgz|bz2|xz|7z|rar|zst)$/i, FileArchive], + [/\.(mp4|mov|webm|mkv|avi)$/i, FilePlay], + [/\.(mp3|wav|ogg|flac|m4a)$/i, FileAudio], + [/\.(py|ts|js|go|rs|sh|sql|java|rb|php|ya?ml|toml|html|css|xml)$/i, FileCode], + [/\.(txt|md|log|pdf|docx?|rtf)$/i, FileText] + ] + function fileIconFor(name: string): IconType { + return FILE_ICONS.find(([re]) => re.test(name))?.[1] ?? FileIcon + } + function nestingLevelOf(key: string): number { const slashes = (key.match(/\//g) ?? []).length return (key.endsWith('/') ? slashes - 1 : slashes) * 2 @@ -1047,41 +1159,50 @@ }) +{#snippet tableLoading()} +
+ Loading table... +
+{/snippet} + {#if workspaceSettingsInitialized === false} - {#if fromWorkspaceSettings} - -
-

Double check the S3 resource fields and try again.

-
- - {:else if s3ResourcePath} - -
-

- Double check the resource fields and that its object storage is reachable, then try again. -

-
- - {:else} - -
-

- The workspace needs to be connected to an S3 storage to use this feature. You can configure it here. -

-
- - {/if} +
+ {#if fromWorkspaceSettings} + +
+

Double check the S3 resource fields and try again.

+
+ + {:else if s3ResourcePath} + +
+

+ Double check the resource fields and that its object storage is reachable, then try + again. +

+
+ + {:else} + +
+

+ The workspace needs to be connected to an S3 storage to use this feature. You can configure it here. +

+
+ + {/if} +
{:else} {#if fileListUnavailable == true} {#if replaceUnauthorizedWarning} {@render replaceUnauthorizedWarning()} {:else} -
+

You don't have access to the S3 bucket resource and your administrator has restricted @@ -1098,28 +1219,25 @@

{/if} {/if} -
+ {#if !fileListUnavailable} -
+ {#if !rootPath} -
- +
{/if} {#if displayedFileKeys.length === 0} {#if fileListLoading} -
-
- Loading content -
+
+ + Loading...
{:else} -
+
{#if filter.trim() !== ''} No files starting with "{filter.trim()}" {:else} @@ -1131,19 +1249,19 @@ can remove every match. Without this the only control that could resume the listing would be hidden behind the empty state. -->
-
{/if} {/if} {:else} -
+
{#snippet header()}{/snippet} {#snippet footer()}{/snippet} @@ -1155,199 +1273,249 @@ {@const load_more_prefix = is_load_more ? item_key.slice(0, -1) : ''} {@const file_info = allFilesByKey[item_key]} -
+
{#if is_load_more} {@const loadMoreNesting = nestingLevelOf(load_more_prefix + 'x') - 2 * rootPathNestingLevel} {@const loadingMore = folderState[load_more_prefix]?.loading === true} - - -
!loadingMore && loadMore(load_more_prefix)} - class={twMerge( - 'flex flex-row h-full text-xs items-center justify-start text-secondary', - loadingMore ? 'cursor-default' : 'cursor-pointer' - )} + disabled={loadingMore} + class="w-full text-xs font-normal flex gap-2 items-center h-8 pr-1 text-secondary hover:bg-surface-hover cursor-pointer disabled:cursor-default disabled:hover:bg-transparent" + style={treeRowIndent(loadMoreNesting)} > -
- -
- {#if loadingMore} - - {:else} - - {/if} -
-
Load more
-
-
+ + {#if loadingMore} + + {:else} + + {/if} + Load more + {:else if file_info} {@const nestingLevel = file_info.nestingLevel - 2 * rootPathNestingLevel} - - -
selectItem(index)} + title={file_info.full_key.slice(rootPath.length)} class={twMerge( - 'flex flex-row h-full font-semibold text-xs items-center justify-start', - selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key - ? 'bg-surface-hover' - : '' + 'w-full text-xs font-normal text-primary flex gap-2 items-center h-8 pr-1 cursor-pointer', + isSelected ? 'bg-surface-secondary' : 'hover:bg-surface-hover' )} + style={treeRowIndent(nestingLevel)} > -
- {#if file_info.type === 'folder'} - {#if folderState[file_info.full_key]?.loading} - - {:else if file_info.collapsed}{:else}{/if} -
- - {#if file_info.display_name === ''} - (empty name) - {:else}{file_info.display_name}{/if} - {#if !lazyMode} - ({file_info.count}{count % 1000 === 0 && - lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name - ? '+' - : ''} item{file_info.count === 1 ? '' : 's'}) - {/if} -
+ {#if isFolder} + {#if folderState[file_info.full_key]?.loading} + {:else} - -
- {file_info.display_name} -
+ {/if} -
-
+ {:else} + + {/if} + + + + {#if file_info.display_name === ''} + (empty name) + {:else}{file_info.display_name}{/if} + + + {#if isFolder && !lazyMode} + + {file_info.count}{count % 1000 === 0 && + lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name + ? '+' + : ''} + + {/if} + {/if}
{/snippet}
{#if fileListLoading === true} -
- Loading content +
+ Loading...
{:else if lazyMode}
{displayedCount} item{displayedCount === 1 ? '' : 's'} shown
{:else} -
+
{displayedCount}{flatHasMore ? '+' : ''} {displayedCount !== count ? 'filtered ' : ''}items (including inside folders)
{#if flatHasMore} - {/if} {/if}
{/if} -
+ {/if} -
- {#if fileMetadata === undefined} -
- {#if fileInfoLoading} -
- {:else if fileListUnavailable} -
- {:else} -
- {/if} -
- {:else} -
-
-

- {((p) => (p.startsWith(rootPath) ? p.slice(rootPath.length) : p))( - fileMetadata.fileKey - )} -

- {#if filePreview !== undefined && (!hideS3SpecificDetails || !readOnlyMode || allowDelete)} -
- {#if !hideS3SpecificDetails} - {@const downloadApiPath = `/w/${ws}/job_helpers/download_s3_file?file_key=${encodeURIComponent(fileMetadata?.fileKey ?? '')}${storage ? `&storage=${storage}` : ''}${s3ResourcePath ? `&s3_resource_path=${encodeURIComponent(s3ResourcePath)}` : ''}`} - {@const downloadName = - fileMetadata?.fileKey.split('/').pop() ?? 'unnamed_download.file'} - {#if shouldDownloadViaClient()} -
+ {:else if !csvOptions} +
+ {#if filePreview.contentType === 'Csv'} + Previewing a {filePreview.contentType?.toLowerCase()} file. Separator character: +
+ +
+ Header row: +
+ +
+ {:else} + Previewing a {filePreview.contentType?.toLowerCase()} file. + {/if} +
+ {/if} 
{#if !emptyString(filePreview.contentPreview)}{filePreview.contentPreview}{:else if filePreview.contentType !== undefined}Preview impossible.{/if}
diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 62f9851fcc..601bf77799 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -72,7 +72,6 @@ Tag, X } from 'lucide-svelte' - import { base } from '$lib/base' import { useLocalStorageValue } from '$lib/svelte5Utils.svelte' import { parsePipelineAnnotations } from './assets/AssetGraph/parsePipelineAnnotations' import DropdownV2 from './DropdownV2.svelte' @@ -93,8 +92,11 @@ import { writable } from 'svelte/store' import { defaultScriptLanguages, processLangs } from '$lib/scripts' import DefaultScripts from './DefaultScripts.svelte' - import { getContext, onDestroy, onMount, setContext, tick, untrack } from 'svelte' + import { getContext, onDestroy, onMount, setContext, tick, untrack, type Snippet } from 'svelte' import EditorHeader from './EditorHeader.svelte' + import PageHeaderContent from '$lib/components/PageHeaderContent.svelte' + import PathEditPopover from '$lib/components/PathEditPopover.svelte' + import { pageHeader, PHONE_BAR } from '$lib/components/pageHeaderRegistry.svelte' import ScriptSettingsBadges from './ScriptSettingsBadges.svelte' import Badge from './common/badge/Badge.svelte' import Modal from './common/modal/Modal.svelte' @@ -125,8 +127,9 @@ import WacExportDrawer from './scripts/WacExportDrawer.svelte' import { UserDraft } from '$lib/userDraft.svelte' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' - import { useTriggerLock } from '$lib/operatorWriteRights' + import { useEditRights, useTriggerLock } from '$lib/operatorWriteRights' const triggerLock = useTriggerLock() + const editRights = useEditRights() let { script = $bindable(), @@ -162,12 +165,14 @@ loadedFromDraft = false, othersDraftsCount = 0, onOpenOthersDrafts, - condensedHeader = false + condensedHeader = false, + ownsPageHeader = false }: ScriptBuilderProps = $props() // Top-bar button size + bar height. Condensed (session preview) uses the // smallest well-supported unified size (`sm`) so the bar is thinner. - const headerBtnSize = $derived(condensedHeader ? 'sm' : 'md') + // In the page header the buttons ride a 44px band, which takes the same size. + const headerBtnSize = $derived(condensedHeader || ownsPageHeader ? 'sm' : 'md') export function getInitialAndModifiedValues(): SavedAndModifiedValue { return { @@ -196,8 +201,24 @@ // right group (hide the ~200px tag select, icon-only Diff/Settings) before the // full group crowds the path into heavy truncation; ~900 is where the path // keeps a usable width given the right group's natural ~440px. + /** Held by the rename popover while it is open; the band's trail reads it so the path does not + * reflow under the pointer as the user types. */ + let pathSnapshot = $state(undefined) + let topbarWidth = $state(0) - const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 900) + // In the page header the buttons share the row with the breadcrumb, so they collapse sooner: + // the trail and the summary take ~560px before either truncates, and the full button group is + // ~515px — about 1100px between them, plus slack for a longer path than most. + const compactBelow = $derived(ownsPageHeader ? 1200 : 900) + const compactTopbar = $derived.by(() => { + const w = ownsPageHeader ? pageHeader.barWidth : topbarWidth + return w > 0 && w < compactBelow + }) + // A phone's bar holds the trail and one button. Diff and Settings join the menu that compact + // already shows, and Deploy stays out: it is what this page is for. + const phoneTopbar = $derived( + ownsPageHeader && pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR + ) // The workspace this editor operates on: deploy, save-draft, trigger loading // and the AutosaveIndicator all target it. Falls back to the full-page @@ -220,7 +241,21 @@ function getCompactMenuItems(): Item[] { const hasTags = (scriptWorkerTags?.length ?? 0) > 0 + const isDraftOnly = savedScript?.no_deployed === true return [ + ...(phoneTopbar && customUi?.topBar?.diff != false + ? [ + { + displayName: 'Diff', + icon: DiffIcon, + disabled: !savedScript || !diffDrawer || isDraftOnly, + action: () => openDiffDrawer() + } + ] + : []), + ...(phoneTopbar && customUi?.topBar?.settings != false + ? [{ displayName: 'Settings', icon: Settings, action: () => (metadataOpen = true) }] + : []), ...(customUi?.topBar?.tagEdit != false && hasTags ? [ { @@ -561,6 +596,7 @@ | 'powershell' | 'bunnative' | 'claudesandbox' + | 'pisandbox' | 'wac_python' | 'wac_typescript' | 'ci_test_bun' @@ -1349,7 +1385,7 @@ onCanceled={() => (perpetualRunsToConfirm = undefined)} /> -{#if !actingUser?.operator} +{#if editRights.roleCanAuthor('script')} @@ -1562,6 +1600,21 @@ > Claude Sandbox + - {#if customUi?.topBar?.path != false} -
- onNavigate?.(item)} - /> -
+ {#if ownsPageHeader} + + + + {:else} +
+
+
+ {@render languageButton(condensedHeader ? 18 : 24)} + {#if customUi?.topBar?.path != false} +
+ onNavigate?.(item)} + /> +
+ {/if} + {@render autosaveIndicator()} + {#if !condensedHeader} + {@render settingsBadges()} + {/if} +
+ + +
+ + {#if $enterpriseLicense && initialPath != '' && !inSessionPane} + {/if} - {#if opWorkspace} - - {/if} - {#if !condensedHeader} - {@const canOpenRuntime = - customUi?.topBar?.settings != false && - customUi?.settingsPanel?.disableRuntime !== true} - { - selectedTab = 'runtime' - metadataOpen = true - } - : undefined} + + +
+ + {@render rightButtons()} +
+
+ {/if} + + {#snippet scriptSummary()} + + {#if script} + +
+ {#snippet summaryText()} + {emptyString(script.summary) ? 'Add a summary...' : script.summary} + {/snippet} + {#if customUi?.topBar?.editablePath != false || customUi?.topBar?.editableSummary != false} + + {:else} + {@render summaryText()} {/if}
+ {/if} + {/snippet} - -
+ + {#snippet scriptPathTrigger(pathLabel: Snippet, triggerClass: string)} + + {#if script} + + {:else} + {@render pathLabel()} + {/if} + {/snippet} + {#snippet scriptMarks()} + {#if script} + + {@render settingsBadges('pl-1.5')} + {/if} + {/snippet} + + {#snippet scriptHeaderActions()} + {#if script} + + {@render autosaveIndicator()} {#if $enterpriseLicense && initialPath != '' && !inSessionPane} {/if} + {@render rightButtons()} + {/if} + {/snippet} - -
+ {#snippet languageButton(size: number)} + + {/snippet} - {#snippet settingsButton()} - {#if customUi?.topBar?.settings != false} - - {/if} - {/snippet} - {#snippet diffButton()} - {#if customUi?.topBar?.diff != false} - {@const isDraftOnly = savedScript?.no_deployed === true} - {@const diffDisabled = !savedScript || !diffDrawer || isDraftOnly} - {@const diffTitle = isDraftOnly - ? 'Deploy this script once to compare against the deployed version' - : 'Diff'} - -
- -
- {/if} - {/snippet} - {#if compactTopbar} - - {#snippet buttonReplacement()} - +
+ {/if} + {/snippet} + {#if compactTopbar} + + {#snippet buttonReplacement()} +
-
+ handleEditScript(false, detail)} + /> + {/snippet} {#if showPipelineHint}
@@ -2417,7 +2604,7 @@ -- materialize to materialize its result, or build it in the - pipeline editor. + pipeline editor. {/if} {#if customUi?.previewPanel?.hideArgs} - -
- {#if testJob?.id && testJob.type === 'CompletedJob' && opWs} - -
- - {testJob.id.slice(0, 8)}… ↗ - - -
- {/if} -
+ +
+ +
+
{#if testIsLoading} {@render cancelTestButton('sm', 'shadow-md')} {:else if (customUi?.previewPanel?.downstreamSubscribers ?? 0) > 0 || customUi?.previewPanel?.onBoundedRun} @@ -2270,6 +2242,31 @@ {@render runTestButton('sm', 'shadow-md')} {/if}
+ {#if testJob?.id && testJob.type === 'CompletedJob' && opWs} + +
+ + {testJob.id.slice(0, 8)}… ↗ + + +
+ {/if} +
{#if customUi?.previewPanel?.argsAboveLogs && schema?.properties && Object.keys(schema.properties).length > 0}
Enable Windmill AI in the diff --git a/frontend/src/lib/components/ScriptPicker.svelte b/frontend/src/lib/components/ScriptPicker.svelte index 781cb5155e..97ac81713d 100644 --- a/frontend/src/lib/components/ScriptPicker.svelte +++ b/frontend/src/lib/components/ScriptPicker.svelte @@ -1,7 +1,6 @@ {#if badges.length > 0} -
+
{#each badges as badge (badge.key)} diff --git a/frontend/src/lib/components/ServiceLogsInner.svelte b/frontend/src/lib/components/ServiceLogsInner.svelte index 85c6cf7fb9..03cd5bf676 100644 --- a/frontend/src/lib/components/ServiceLogsInner.svelte +++ b/frontend/src/lib/components/ServiceLogsInner.svelte @@ -10,8 +10,10 @@ import Toggle from './Toggle.svelte' import TextInput from './text_input/TextInput.svelte' import Popover from './Popover.svelte' + // The clickable one, for the folded switches; the import above is the hover tooltip. + import MeltPopover from './meltComponents/Popover.svelte' import { sendUserToast } from '$lib/toast' - import { onDestroy, tick, type Snippet } from 'svelte' + import { onDestroy, tick } from 'svelte' import { fade } from 'svelte/transition' import { AlertTriangle, @@ -22,7 +24,8 @@ Loader2, ScrollText, Search, - SearchX + SearchX, + SlidersHorizontal } from 'lucide-svelte' import { copyToClipboard, scroll_into_view_if_needed_polyfill, truncateRev } from '$lib/utils' import LogSnippetViewer from './LogSnippetViewer.svelte' @@ -33,15 +36,24 @@ import { goto } from '$lib/navigation' import { page } from '$app/state' import { watch } from 'runed' + import Tooltip from './Tooltip.svelte' + import PageHeaderContent from './PageHeaderContent.svelte' + import { pageHeader } from './pageHeaderRegistry.svelte' interface Props { searchTerm: string tagLabel?: string - /** Rendered at the start of the toolbar row, so the page title shares it. */ - title?: Snippet } - let { searchTerm = $bindable(), tagLabel, title }: Props = $props() + let { searchTerm = $bindable(), tagLabel }: Props = $props() + + /** Bar width below which the two switches fold into a popover. Measured: the breadcrumb takes + * ~360, the search field asks for 384 and the timeframe 223, so under this the field is what + * would start giving width back — and it is the control the page exists for. */ + const COMPACT_SWITCHES_BAR = 1200 + const compactSwitches = $derived( + pageHeader.barWidth > 0 && pageHeader.barWidth < COMPACT_SWITCHES_BAR + ) let queryParseErrors: string[] | undefined = $state(undefined) @@ -692,88 +704,135 @@ -
-
- {@render title?.()} -
- - - {#if searchTerm !== '' && queryParseErrors && queryParseErrors.length > 0} -
- - - {#snippet text()} -
- Some search terms were ignored because they could not be parsed: -
    - {#each queryParseErrors ?? [] as msg} -
  • {msg}
    • - {/each} -
-
- {/snippet} - -
- {/if} -
- { - minTs = undefined - maxTs = undefined - allLogs = undefined - const ts = timeframe.computeMinMax() - getAllLogs(ts.minTs ?? undefined, ts.maxTs ?? undefined) +{#snippet serviceLogsHint()} + + Logs written by the servers, workers and indexers of this instance, grouped by host. + +{/snippet} + +{#snippet serviceLogsActions()} + +
+ + -
- { - allLogs = undefined - getAllLogs(minTs, maxTs) - }} - /> - { - if (e.detail) { - getAllLogs(maxTs, undefined) - } else { - timeout && clearTimeout(timeout) - } - }} - options={{ - right: 'Auto-refresh', - rightTooltip: 'Fetch new log files every 5 seconds. Paused while searching.' - }} - /> -
+ {#if searchTerm !== '' && queryParseErrors && queryParseErrors.length > 0} +
+ + + {#snippet text()} +
+ Some search terms were ignored because they could not be parsed: +
    + {#each queryParseErrors ?? [] as msg} +
  • {msg}
    • + {/each} +
+
+ {/snippet} + +
+ {/if}
+ + { + minTs = undefined + maxTs = undefined + allLogs = undefined + const ts = timeframe.computeMinMax() + getAllLogs(ts.minTs ?? undefined, ts.maxTs ?? undefined) + }} + /> + + {#if compactSwitches} + + {#snippet trigger()} + -
-
-
+ + {/if} + +{#snippet draftsAction()} + + {#if phone} + + {:else} + + {draftCount} draft{draftCount !== 1 ? 's' : ''} + + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/WorkspaceItemKindIcon.svelte b/frontend/src/lib/components/WorkspaceItemKindIcon.svelte new file mode 100644 index 0000000000..9d07f6ef89 --- /dev/null +++ b/frontend/src/lib/components/WorkspaceItemKindIcon.svelte @@ -0,0 +1,19 @@ + + + +{#if kind === 'flow'} + +{:else if kind === 'app'} + +{:else} + +{/if} diff --git a/frontend/src/lib/components/WorkspaceScopeTrigger.svelte b/frontend/src/lib/components/WorkspaceScopeTrigger.svelte index 77fe52b421..37a27f3812 100644 --- a/frontend/src/lib/components/WorkspaceScopeTrigger.svelte +++ b/frontend/src/lib/components/WorkspaceScopeTrigger.svelte @@ -2,8 +2,8 @@ import { userWorkspaces, type UserWorkspace } from '$lib/stores' import { findWorkspaceRoot } from '$lib/utils/workspaceHierarchy' import { forkAccentStyle } from '$lib/utils/forkColor' - import { Badge, Button } from '$lib/components/common' - import { devBadgeText } from '$lib/utils/devWorkspaceLabel' + import { Button } from '$lib/components/common' + import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { Building, ChevronDown, EllipsisVertical, GitFork } from 'lucide-svelte' import DropdownV2 from '$lib/components/DropdownV2.svelte' import { type Item } from '$lib/utils' @@ -171,12 +171,12 @@ {#if pendingFork} (new) {:else if currentWs?.is_dev_workspace} - {devBadgeText(currentWs.dev_workspace_label)} + {/if} {/if} diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte index d00e52d77e..e87745887b 100644 --- a/frontend/src/lib/components/aiEvals/EvalScorers.svelte +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -21,7 +21,6 @@ Settings, Trash2 } from 'lucide-svelte' - import { base } from '$lib/base' import AddScorer from './AddScorer.svelte' import { kindLabel, parseThreshold, scorerLabel, type ScorerKind } from './evalUtils' @@ -304,8 +303,8 @@ - import { base } from '$lib/base' import { Button } from '$lib/components/common' import Select from '$lib/components/select/Select.svelte' import DataTable from '$lib/components/table/DataTable.svelte' @@ -768,7 +767,7 @@ {#if experiment?.run_job_id} Open the job @@ -962,7 +961,7 @@ {#if openRow.job_id} Open the case job diff --git a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts index 21371aa08b..ae3a82e97c 100644 --- a/frontend/src/lib/components/apps/components/display/dbtable/utils.ts +++ b/frontend/src/lib/components/apps/components/display/dbtable/utils.ts @@ -41,6 +41,8 @@ export type ColumnDef = { children?: ColumnDef[] // DBExplorer ignored?: boolean + /** False when the column cannot be filtered on, e.g. a nested value. */ + filterable?: boolean hideInsert?: boolean editable?: boolean overrideDefaultValue?: boolean diff --git a/frontend/src/lib/components/apps/components/helpers/eval.ts b/frontend/src/lib/components/apps/components/helpers/eval.ts index bacfb65b2d..0b6bdd55db 100644 --- a/frontend/src/lib/components/apps/components/helpers/eval.ts +++ b/frontend/src/lib/components/apps/components/helpers/eval.ts @@ -1,7 +1,6 @@ import type { World } from '../../rx' import { sendUserToast } from '$lib/toast' import { waitJob } from '$lib/components/waitJob' -import { base } from '$lib/base' import { appNavigateSameWindow } from '../../utils' import { OpenAPI } from '$lib/gen/core/OpenAPI' @@ -312,10 +311,10 @@ export async function eval_like( params.append('s3', input.s3 ?? '') if (input.storage) params.append('storage', input.storage) params.append('token', OpenAPI.TOKEN) - const s3href = `${base}/api/w/${workspaceId}/apps_u/download_s3_file/${appPath}?${params.toString()}` + const s3href = `/api/w/${workspaceId}/apps_u/download_s3_file/${appPath}?${params.toString()}` downloadFile(s3href, filename || input.s3) } else { - const s3href = `${base}/api/w/${workspaceId}/job_helpers/download_s3_file?file_key=${encodeURIComponent( + const s3href = `/api/w/${workspaceId}/job_helpers/download_s3_file?file_key=${encodeURIComponent( input?.s3 ?? '' )}${input?.storage ? `&storage=${input.storage}` : ''}` downloadFile(s3href, filename || input.s3) diff --git a/frontend/src/lib/components/apps/editor/AppEditor.svelte b/frontend/src/lib/components/apps/editor/AppEditor.svelte index 073d6c3e7d..ac0d2b1cb6 100644 --- a/frontend/src/lib/components/apps/editor/AppEditor.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditor.svelte @@ -86,7 +86,8 @@ othersDraftsCount = 0, onOpenOthersDrafts, onRestore, - onDeploy + onDeploy, + ownsPageHeader = false }: AppEditorProps = $props() migrateApp(untrack(() => app)) @@ -885,6 +886,7 @@ {#if !$userStore?.operator} {#if $appStore} hideBottomPanel()} /> {#if $mode === 'preview'} - +
{/if} - +
void + /** True for the route's own editor: its top bar becomes the page header. */ + ownsPageHeader?: boolean } let { @@ -142,10 +147,11 @@ othersDraftsCount = 0, onOpenOthersDrafts, onRestore, - onDeploy + onDeploy, + ownsPageHeader = false }: Props = $props() - /** Mirror of the path the user is editing in the pen popover. Initialized + /** Mirror of the path the user is editing in the rename popover. Initialized * once from `newPath` (or a synthesized path for new apps) and only * updated by user input from then on — we deliberately do NOT sync from * `newPath` afterwards so the user's in-flight rename isn't clobbered by @@ -223,6 +229,18 @@ // the preview to the deployed version. const inSessionPane = !!getContext('aiChatManager') + // The header's buttons render under the page header, not under this component, so the contexts + // they look up have to travel with them. The app's own two carry everything the canvas toggles, + // the panel buttons, Debug runs and the preview switch read. + /** Held by the rename popover while it is open; the band's trail reads it so the path does not + * reflow under the pointer as the user types. */ + let pathSnapshot = $state(undefined) + + const headerContexts = new Map([ + ['AppViewerContext', getContext('AppViewerContext')], + ['AppEditorContext', getContext('AppEditorContext')] + ]) + const loading = $state({ publish: false, save: false @@ -243,19 +261,44 @@ // Top-bar responsive collapse — container width, not viewport. let topbarWidth = $state(0) - const compactTopbar = $derived(topbarWidth > 0 && topbarWidth < 720) + // In the page header the buttons share the row with the breadcrumb, so they shed the canvas + // toggles and fold Debug runs into the menu sooner: the whole group is ~840px and the trail + // with the summary takes ~430px of the same row. + // Measured on the band rather than estimated: with the canvas toggles shown the actions alone + // are 826-911px and the trail with the summary another ~500, so the row overflows the bar at + // every width up to about 1470 — 1350 was not high enough either. Below this the toggles stand + // down and the menu carries all three, which is what keeps them reachable rather than merely + // out of the way. + const compactBelow = $derived(ownsPageHeader ? 1500 : 720) + const compactTopbar = $derived.by(() => { + const w = ownsPageHeader ? pageHeader.barWidth : topbarWidth + return w > 0 && w < compactBelow + }) + // A phone's bar holds the trail and Deploy. Which panels are open, the export and the + // editor/preview switch join the menu compact already shows. + const phoneTopbar = $derived( + ownsPageHeader && pageHeader.barWidth > 0 && pageHeader.barWidth < PHONE_BAR + ) function closeSaveDrawer() { saveDrawerOpen = false } + // `parent_version` is the draft's stale-check base. Deployed, it would put a version + // id that changes on every deploy into git-synced files and version diffs. + function withoutDraftBase(a: T): T { + if (!a) return a + const { parent_version: _pv, ...rest } = a + return rest as T + } + async function createApp(path: string) { policy = await updatePolicy($app, policy) try { await AppService.createApp({ workspace: $workspaceStore!, requestBody: { - value: $app, + value: withoutDraftBase($app), path, summary: $summary, policy, @@ -314,7 +357,7 @@ orderedJsonStringify( replaceFalseWithUndefined({ summary: $summary, - value: $app, + value: withoutDraftBase($app), path: newEditedPath || savedApp.path, policy, custom_path: customPath, @@ -357,6 +400,7 @@ // Strip off extra information deployedValue = replaceFalseWithUndefined({ ...deployedApp, + value: withoutDraftBase(deployedApp.value as App), id: undefined, created_at: undefined, created_by: undefined, @@ -371,7 +415,7 @@ workspace: $workspaceStore!, path: $appPath!, requestBody: { - value: $app!, + value: withoutDraftBase($app!), summary: $summary, policy, path: npath, @@ -590,7 +634,49 @@ selectedJobId = $jobs[$jobs.length - 1] } $jobsDrawerOpen = true - }, + } + }, + // The canvas toggles stand down with the bar, and the breakpoint among them is the + // only way onto the mobile layout — so each one keeps an entry here. + { + displayName: $breakpoint === 'sm' ? 'Computer view' : 'Mobile view', + icon: $breakpoint === 'sm' ? Laptop2 : Smartphone, + action: () => ($breakpoint = $breakpoint === 'sm' ? 'lg' : 'sm') + }, + // The setting the mobile view is edited for, which otherwise lives beside the + // toggle group that stood down: without it here a deployed app could be put on + // the mobile layout at this width and never taken off it. + ...($breakpoint === 'sm' + ? [ + { + displayName: $app?.mobileViewOnSmallerScreens + ? 'Mobile view on small screens: on' + : 'Mobile view on small screens: off', + icon: Smartphone, + action: () => { + if ($app) $app.mobileViewOnSmallerScreens = !$app.mobileViewOnSmallerScreens + } + } + ] + : []), + { + displayName: $app?.fullscreen ? 'Centered canvas' : 'Full-width canvas', + icon: $app?.fullscreen ? AlignHorizontalSpaceAround : Expand, + action: () => { + if ($app) $app.fullscreen = !$app.fullscreen + } + }, + { + displayName: + $app?.darkMode === undefined + ? 'Theme: automatic' + : $app.darkMode + ? 'Theme: dark' + : 'Theme: light', + icon: $app?.darkMode === undefined ? SunMoon : $app.darkMode ? Moon : Sun, + // Cycles auto → light → dark, the order the toggle group reads in. + action: () => + setTheme($app?.darkMode === undefined ? false : $app.darkMode ? undefined : true), separatorBottom: true } ] @@ -676,7 +762,7 @@ deployed: deployedValue ?? savedApp, current: { summary: $summary, - value: $app, + value: withoutDraftBase($app), path: newEditedPath || savedApp.path, policy, custom_path: customPath, @@ -770,7 +856,7 @@ bind:deployedValue currentValue={{ summary: $summary, - value: $app, + value: withoutDraftBase($app), path: newEditedPath || savedApp?.path, policy, custom_path: customPath, @@ -821,7 +907,7 @@ deployed: deployedValue ?? savedApp, current: { summary: $summary, - value: $app, + value: withoutDraftBase($app), path: newEditedPath || savedApp.path, policy, custom_path: customPath, @@ -913,141 +999,232 @@ -
- + ` placeholder, and the trail would name that instead of the path Deploy + // will create. Same fallback chain the rename sites read. + // Frozen while the rename popover is open so the trail holds still as the user types. + kind: 'app', + path: pathSnapshot ?? (newEditedPath || $appPath || newPath || undefined), + summaryContent: appSummary, + pathTrigger: appPathTrigger + }} + actions={appHeaderActions} + contexts={headerContexts} + separator="always" + /> +{:else} +
+ -
-
- (onNavigate ? onNavigate(item) : goto(editPathFor(item)))} - /> -
-
- {#if $app} - { - $app.fullscreen = detail === 'true' - }} - > - {#snippet children({ item })} - - - {/snippet} - - {/if} - {#if $app} - { - const theme = detail === 'dark' ? true : detail === 'sun' ? false : undefined - setTheme(theme) - }} - selected={$app.darkMode === undefined ? 'auto' : $app.darkMode ? 'dark' : 'sun'} - > - {#snippet children({ item })} - - - - {/snippet} - - {/if} -
- - {#snippet children({ item })} - - - {#if $breakpoint === 'sm'} - - {/if} - {/snippet} - -
-
- {#if $workspaceStore} -
- +
+ (onNavigate ? onNavigate(item) : goto(editPathFor(item)))} />
- {/if} -
+ {@render canvasToggles()} + {@render autosaveIndicator()} +
+ {@render panelButtons()} + {@render awarenessMark()} + {@render appActions()} +
+{/if} + + +{#snippet appPathTrigger(pathLabel: Snippet, triggerClass: string)} + +{/snippet} + +{#snippet appSummary()} + +
+ {#snippet summaryText()} + {emptyString($summary) ? 'Add a summary...' : $summary} + {/snippet} + +
+{/snippet} + +{#snippet appHeaderActions()} + +
+ {@render canvasToggles()} + {@render autosaveIndicator()} + {#if !phoneTopbar} + {@render panelButtons()} + {@render awarenessMark()} + {/if} + {@render appActions()} +
+{/snippet} + +{#snippet canvasToggles()} +
+ {#if $app} + { + $app.fullscreen = detail === 'true' + }} + > + {#snippet children({ item })} + + + {/snippet} + + {/if} + {#if $app} + { + const theme = detail === 'dark' ? true : detail === 'sun' ? false : undefined + setTheme(theme) + }} + selected={$app.darkMode === undefined ? 'auto' : $app.darkMode ? 'dark' : 'sun'} + > + {#snippet children({ item })} + + + + {/snippet} + + {/if} +
+ + {#snippet children({ item })} + + + {#if $breakpoint === 'sm'} + + {/if} + {/snippet} + +
+
+{/snippet} + +{#snippet autosaveIndicator()} + {#if $workspaceStore} +
+ +
+ {/if} +{/snippet} + +{#snippet panelButtons()} {#if $mode !== 'preview'}
{/if} +{/snippet} + +{#snippet awarenessMark()} {#if $enterpriseLicense && $appPath != '' && !inSessionPane}
{/if} +{/snippet} + +{#snippet appActions()}
@@ -1131,14 +1314,19 @@
+ - +
-
+{/snippet} diff --git a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte index d062247fa4..7b25dc234e 100644 --- a/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte +++ b/frontend/src/lib/components/apps/editor/AppEditorHeaderDeploy.svelte @@ -3,6 +3,7 @@ import Badge from '$lib/components/common/badge/Badge.svelte' import Toggle from '$lib/components/Toggle.svelte' import { enterpriseLicense, userStore } from '$lib/stores' + import { useOperatorBuilderApps } from '$lib/operatorWriteRights' import { Loader2 } from 'lucide-svelte' import Tooltip from '$lib/components/Tooltip.svelte' @@ -15,7 +16,6 @@ import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' import Path from '$lib/components/Path.svelte' import { computeSecretUrl } from './appDeploy.svelte' - import { base } from '$lib/base' import { isCloudHosted } from '$lib/cloud' import EEOnly from '$lib/components/EEOnly.svelte' import TextInput from '$lib/components/text_input/TextInput.svelte' @@ -34,6 +34,7 @@ const operatingWorkspaceStore = useOperatingWorkspace() const operatingUser = useOperatingUser() const actingUser = $derived(operatingUser.current) + const operatorBuilderApps = useOperatorBuilderApps() const WM_DEPLOYERS_GROUP = 'wm_deployers' @@ -185,7 +186,7 @@ let secretUrl: string | undefined = $state(undefined) let secretUrlHref = $derived(secretUrl ? computeSecretUrl(secretUrl) : undefined) let fullCustomUrl = $derived( - `${window.location.origin}${base}/a/${ + `${window.location.origin}/a/${ isCloudHosted() || globalWorkspacedRoute ? opWs + '/' : '' }${customPath}` ) @@ -364,7 +365,7 @@
{ policy.sandbox = e.detail || undefined // Counted where the toggle is flipped rather than where the policy is @@ -387,8 +388,13 @@ setPublishState(e.detail ? 'Sandbox isolation enabled' : 'Sandbox isolation disabled') } }} - disabled={!savedApp} + disabled={!savedApp || $operatorBuilderApps} /> + {#if $operatorBuilderApps} +
+ Required for apps built by operators, and cannot be turned off. +
+ {/if}
Controls what the app's browser-side code can reach in each viewer's browser — distinct from the on-behalf-of model above (which sets who its runnables run as). Off by default, the app's code diff --git a/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte b/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte index d77deb6579..672d8dd6e6 100644 --- a/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte +++ b/frontend/src/lib/components/apps/editor/AppReportsDrawerInner.svelte @@ -15,7 +15,6 @@ WorkspaceService } from '$lib/gen' import { workspaceStore } from '$lib/stores' - import { base } from '$lib/base' import { emptyString, formatCron, sendUserToast, tryEvery } from '$lib/utils' import SchemaForm from '$lib/components/SchemaForm.svelte' import Button from '$lib/components/common/button/Button.svelte' @@ -644,7 +643,7 @@

The workspace needs to be connected to Slack to use this feature. You can configure it hereconfigure it here.

+ +{#if ownsPageHeader} + +{/if} + + +{#if showEdit && (menuHidden || !ownsPageHeader)} +
+ {@render editAction()}
{/if} + +{#snippet editAction()} + {#if compact} + + + {:else} + {#if hasPublicUrl} + + + {/if} + + + {/if} +{/snippet} diff --git a/frontend/src/lib/components/apps/editor/PreviewToggle.svelte b/frontend/src/lib/components/apps/editor/PreviewToggle.svelte index 7a72ac62b3..4e20301043 100644 --- a/frontend/src/lib/components/apps/editor/PreviewToggle.svelte +++ b/frontend/src/lib/components/apps/editor/PreviewToggle.svelte @@ -8,9 +8,11 @@ interface Props { loading?: boolean + /** Drops the two labels, for a bar with no room for them: the pen and the eye carry it. */ + iconOnly?: boolean } - let { loading = false }: Props = $props() + let { loading = false, iconOnly = false }: Props = $props() const { mode, jobs, jobsById } = getContext('AppViewerContext') @@ -24,7 +26,8 @@ > {#snippet children({ item })} import { applyDarkModeVariant } from '$lib/darkModeVariant' - import { enterpriseLicense, userStore, type UserExt } from '$lib/stores' - import { base } from '$app/paths' - import { page } from '$app/state' - import Login from '$lib/components/Login.svelte' + import type { UserExt } from '$lib/stores' import { isCloudHosted } from '$lib/cloud' - import Alert from '$lib/components/common/alert/Alert.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import WindmillIcon from '$lib/components/icons/WindmillIcon.svelte' - import { getContext, setContext } from 'svelte' - import { - EMBED_NAV_CONTEXT_KEY, - IS_APP_PUBLIC_CONTEXT_KEY, - type EditorBreakpoint, - type EmbedNav - } from '../types' - import { UserService, type AppWithLastVersion, type GlobalWhoamiResponse } from '$lib/gen' - import { urlParamsToObject } from '$lib/utils' - import { goto } from '$app/navigation' - import { loadAppPreview } from './loadAppPreview' + import { setContext } from 'svelte' + import { IS_APP_PUBLIC_CONTEXT_KEY } from '../types' + import type { AppWithLastVersion } from '$lib/gen/types.gen' import RawAppPreview from '$lib/components/raw_apps/RawAppPreview.svelte' import type { Runnable } from '$lib/components/raw_apps/rawAppPolicy' - import { twMerge } from 'tailwind-merge' - import { writable } from 'svelte/store' + import { loadAppCss } from './publicAppApi' let { notExists, @@ -33,6 +20,7 @@ app, workspace, user = undefined, + license = undefined, inWorkspace = false, hideRefreshBar = false, syncHashToUrl = true @@ -46,11 +34,13 @@ onLoginSuccess: () => void app: (AppWithLastVersion & { value: any; workspace_id?: string }) | undefined workspace: string | undefined - /** The viewer's membership in `workspace`, for the app's `ctx`. Without it `ctx` falls - * back to `userStore`, which describes the workspace the page is navigated to — the - * same one on every standalone viewer route, but not inside an AI session's preview - * tab, where the app belongs to the session's workspace. */ + /** The viewer's membership in `workspace`, for the app's `ctx`. Callers pass it + * explicitly — `$userStore` describes the workspace the page is navigated to, which + * is not the app's inside an AI session's preview tab. */ user?: UserExt | undefined + /** The instance license, which fades the "Powered by" badge and lets a low-code app's + * custom CSS apply. */ + license?: string | undefined /** * In-workspace rendering (`/apps/get`, `/app_embed`): keep exact parity * with the pre-sandbox member viewer — no "Powered by Windmill" badge, no @@ -65,23 +55,11 @@ // Use workspace from props or from app.workspace_id (for custom path responses) let effectiveWorkspace = $derived(workspace ?? app?.workspace_id) - // `ctx.username` / `ctx.groups` are per-workspace, and they sit next to `ctx.workspace` - // in the same object, so they have to come from the same workspace it names. - let ctxUser = $derived(user ?? $userStore) - // On the public surfaces (untrusted distribution) runnable-authored html/svg needs // the viewer's approval before it renders, unless the app sandbox isolates it. The // in-workspace viewer renders it verbatim. See getAppMarkupTrust. setContext(IS_APP_PUBLIC_CONTEXT_KEY, !inWorkspace) - // WIN-2006: inside the opaque viewer iframe, navigations to other routes - // (navbar "app" items) must happen on the TOP page — the iframe is cookieless, - // so navigating it would just show a login screen. PublicAppFrame provides the - // relay; outside the opaque viewer this is undefined and goto works directly. - const embedNav = getContext(EMBED_NAV_CONTEXT_KEY) - - const breakpoint = writable('lg') - const darkMode = window.localStorage.getItem('dark-mode') ?? (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light') @@ -93,135 +71,75 @@ } // This route bypasses the (root) layout, so restore the variant class too. applyDarkModeVariant() - - let globalUser = $state(undefined) - async function loadGlobalUser() { - try { - globalUser = await UserService.globalWhoami() - } catch (error) { - console.error(error) - } - // const user = await fetch('/api/global/user') - // console.log(user) - } - - // Only the no-access page reads it, to tell a signed-in non-member which - // workspace the app belongs to. - $effect(() => { - if (noPermission && !guestAppPath && !$userStore && !globalUser) { - loadGlobalUser() - } - }) + {#if !inWorkspace} -
- Powered by   Windmill -
+ Powered by   Windmill +
+ {/await} {/if} -{#if notExists} -
There was an error loading the app, is the url correct? Go to Windmill -
-{:else if noPermission} - {#if guestAppPath && !$userStore} -
Sign in to open this app
-
- You do not need a Windmill account. Signing in lets you open this app and nothing else. -
- {:else} -
- This app requires read access -
-
- {#if $userStore}You are logged in but have no read access to this app{:else if globalUser && effectiveWorkspace} - You are logged in but are not a member of the workspace {effectiveWorkspace} this app is part of - {:else}You must be logged in and have read access to this app{/if}
- {/if} -
- {#if !jwtError} - - {/if} -
+{#if notExists || noPermission} + {#await Promise.all([loadAppCss(), import('./PublicAppAccess.svelte')]) then [, { default: PublicAppAccess }]} + + {/await} {:else if app} {#key app} {#if app.raw_app && effectiveWorkspace} - } - {syncHashToUrl} - /> + +
+ } + {syncHashToUrl} + /> +
{:else if app.raw_app && !effectiveWorkspace} -
- - Unable to load raw app: workspace information is missing. - -
+ {#await Promise.all([loadAppCss(), import('$lib/components/common/alert/Alert.svelte')]) then [, { default: Alert }]} +
+ + Unable to load raw app: workspace information is missing. + +
+ {/await} {:else} -
- {#await loadAppPreview()} - - {:then Module} - goto(path)} - gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} - /> - {:catch} -
- Reload the page to try again. -
- {/await} -
+ {#await Promise.all([loadAppCss(), import('./PublicLowCodeApp.svelte')]) then [, { default: PublicLowCodeApp }]} + + {/await} {/if} {/key} {:else} - + {#await loadAppCss() then} + + {/await} {/if} diff --git a/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte b/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte new file mode 100644 index 0000000000..5e1d80b893 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/PublicAppAccess.svelte @@ -0,0 +1,80 @@ + + +{#if notExists} +
There was an error loading the app, is the url correct? Go to Windmill +
+{:else} + {#if guestAppPath && !user} +
Sign in to open this app
+
+ You do not need a Windmill account. Signing in lets you open this app and nothing else. +
+ {:else} +
+ This app requires read access +
+
+ {#if user}You are logged in but have no read access to this app{:else if globalUser && workspace} + You are logged in but are not a member of the workspace {workspace} this app is part of + {:else}You must be logged in and have read access to this app{/if}
+ {/if} +
+ {#if !jwtError} + { + dropPrefetched() + onLoginSuccess() + }} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> + {/if} +
+{/if} diff --git a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte index e994767488..eacb3652e1 100644 --- a/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte +++ b/frontend/src/lib/components/apps/editor/PublicAppFrame.svelte @@ -23,19 +23,42 @@ * bearer token (no cookie); the raw wrapper document always carries `CSP: sandbox`. */ import { BROWSER } from 'esm-env' - import { OpenAPI, UserService } from '$lib/gen' + import { OpenAPI } from '$lib/gen/core/OpenAPI' import { page } from '$app/state' import { onDestroy, onMount, setContext, type Snippet } from 'svelte' - import Alert from '$lib/components/common/alert/Alert.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' - import { base } from '$app/paths' import { goto } from '$app/navigation' - import Login from '$lib/components/Login.svelte' - import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils' + import { WINDMILL_RESERVED_QUERY_PARAMS } from '$lib/utils/queryParams' import { EMBED_NAV_CONTEXT_KEY, type EmbedNav } from '../types' import { loadAppPreview } from './loadAppPreview' - import RawAppSdkConsent from '$lib/components/raw_apps/RawAppSdkConsent.svelte' - import { hasStoredSdkConsent, storeSdkConsent } from '$lib/components/raw_apps/sdkScopes' + import { dropPrefetched, loadAppCss } from './publicAppApi' + import { + hasStoredSdkConsent, + sdkConsentCovers, + storeSdkConsent + } from '$lib/components/raw_apps/sdkScopes' + + // Everything but a rendered app (sign-in, errors, the SDK consent prompt) loads on + // demand: a raw app that renders right away needs none of it. See publicAppApi.ts. + let frameUi: + | Promise<{ + Alert: typeof import('$lib/components/common/alert/Alert.svelte').default + Login: typeof import('$lib/components/Login.svelte').default + RawAppSdkConsent: typeof import('$lib/components/raw_apps/RawAppSdkConsent.svelte').default + }> + | undefined + function loadFrameUi() { + return (frameUi ??= Promise.all([ + loadAppCss(), + import('$lib/components/common/alert/Alert.svelte'), + import('$lib/components/Login.svelte'), + import('$lib/components/raw_apps/RawAppSdkConsent.svelte') + ]).then(([, alert, login, consent]) => ({ + Alert: alert.default, + Login: login.default, + RawAppSdkConsent: consent.default + }))) + } type EmbedToken = { token?: string | null @@ -231,7 +254,8 @@ if (offerSignIn && accountSession === 'unknown') { // Workspace-less, so it answers for an account and never for a guest // (pinned to its workspace) or for nobody. - UserService.getCurrentEmail() + import('$lib/gen') + .then(({ UserService }) => UserService.getCurrentEmail()) .then(() => (accountSession = 'held')) .catch(() => (accountSession = 'none')) } @@ -245,8 +269,12 @@ let staleGuestLogoutFailed = $state(false) $effect(() => { if (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared) { - UserService.logout() - .then(() => (staleGuestCleared = true)) + import('$lib/gen') + .then(({ UserService }) => UserService.logout()) + .then(() => { + dropPrefetched() + staleGuestCleared = true + }) .catch(() => (staleGuestLogoutFailed = true)) } }) @@ -388,7 +416,7 @@ appPath = resp.app_path ?? appPath workspaceId = resp.workspace_id ?? workspaceId const granted = resp.sdk_scopes ?? [] - if (!granted.every((s) => approved.includes(s))) { + if (!sdkConsentCovers(approved, granted)) { // The app was redeployed with more scopes between the prompt and the // mint. Never inject a token carrying scopes the viewer wasn't shown: // drop it and ask again with the new set. @@ -423,7 +451,10 @@ // Render the app directly on this origin: same-origin when unsandboxed // (the default), or a single opaque bundle iframe when it's a sandboxed // raw app. A low-code app's runtime downloads alongside the app payload. - if (!isRaw) loadAppPreview().catch(() => {}) + if (!isRaw) { + loadAppCss().catch(() => {}) + loadAppPreview().catch(() => {}) + } onViewerReady?.(undefined, requestTokenRefresh) } else { // Sandboxed low-code: hand the scoped token to the opaque viewer iframe. @@ -533,6 +564,7 @@ onMount(() => { if (isViewer) { // Only a sandboxed low-code app is ever framed as a viewer. + loadAppCss().catch(() => {}) loadAppPreview().catch(() => {}) window.addEventListener('message', handleViewerMessage) installHashRelay() @@ -565,105 +597,120 @@ {#if viewerReady} {@render viewer()} {:else if viewerOrphaned} -
- - This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your own - page, use the app's public URL without the wm_embed parameter. - -
+ {#await loadFrameUi() then { Alert }} +
+ + This is a Windmill app viewer and must be loaded by Windmill. If you embedded it in your + own page, use the app's public URL without the wm_embed parameter. + +
+ {/await} {:else} - + {#await loadAppCss() then} + + {/await} {/if} {:else if status === 'loading'} - -{:else if status === 'notExists' && !offerSignIn} -
- - There was an error loading the app, is the url correct? - Go to Windmill - -
-{:else if status === 'sdkPrompt'} - - -{:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} - -{:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} -
- - The app could not be reached to find out who may open it. Reload to try again. - -
-{:else if offerSignIn} - - {#if signInDidNotHelp} - -
- You are signed in, but this app is not open to you -
-
- It is open to the people it was shared with{guestAppPath - ? ', and to guests who have no Windmill account' - : ''}. Ask the person who shared it to give your account access. -
+ {#await loadAppCss() then} + + {/await} +{:else if status === 'ready'} + {#if unsandboxed} + + {@render viewer()} + {:else if isRaw} + + {@render viewer()} {:else} - {#if guestAppPath} -
Sign in to open this app
-
- You do not need a Windmill account. Signing in lets you open this app and nothing else. -
- {:else} -
- This app requires read access -
- {/if} - {#if signInError} -
- {signInError} -
- {/if} -
- { - signInError = undefined - accountSession = 'unknown' - initEmbedder() - }} - onLoginError={(message) => (signInError = message)} - popup - guestApp={guestAppPath} - rd={page.url.pathname + page.url.search + page.url.hash} - /> -
+ + {/if} -{:else if unsandboxed} - - {@render viewer()} -{:else if isRaw} - - {@render viewer()} {:else} - - + {#await loadFrameUi() then { Alert, Login, RawAppSdkConsent }} + {#if status === 'notExists' && !offerSignIn} +
+ + There was an error loading the app, is the url correct? + Go to Windmill + +
+ {:else if status === 'sdkPrompt'} + + + {:else if offerSignIn && (guestEntry === 'pending' || accountSession === 'unknown' || (deniedStatus === 403 && guestEntry === 'guest' && !staleGuestCleared && !staleGuestLogoutFailed))} + + {:else if offerSignIn && (guestEntry === 'error' || staleGuestLogoutFailed)} +
+ + The app could not be reached to find out who may open it. Reload to try again. + +
+ {:else if offerSignIn} + + {#if signInDidNotHelp} + +
+ You are signed in, but this app is not open to you +
+
+ It is open to the people it was shared with{guestAppPath + ? ', and to guests who have no Windmill account' + : ''}. Ask the person who shared it to give your account access. +
+ {:else} + {#if guestAppPath} +
Sign in to open this app
+
+ You do not need a Windmill account. Signing in lets you open this app and nothing else. +
+ {:else} +
+ This app requires read access +
+ {/if} + {#if signInError} +
+ {signInError} +
+ {/if} +
+ { + dropPrefetched() + signInError = undefined + accountSession = 'unknown' + initEmbedder() + }} + onLoginError={(message) => (signInError = message)} + popup + guestApp={guestAppPath} + rd={page.url.pathname + page.url.search + page.url.hash} + /> +
+ {/if} + {/if} + {/await} {/if} diff --git a/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte b/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte new file mode 100644 index 0000000000..f74be67166 --- /dev/null +++ b/frontend/src/lib/components/apps/editor/PublicLowCodeApp.svelte @@ -0,0 +1,91 @@ + + +
+ {#await loadAppPreview()} + + {:then Module} + goto(path)} + gotoFn={(path, opt) => (embedNav ? embedNav.navigateTop(path) : goto(path, opt))} + /> + {:catch} +
+ Reload the page to try again. +
+ {/await} +
diff --git a/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts b/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts index f799cb864a..5c22e3f475 100644 --- a/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts +++ b/frontend/src/lib/components/apps/editor/appDeploy.svelte.ts @@ -1,7 +1,6 @@ -import { base } from '$lib/base' import { workspaceStore } from '$lib/stores' import { get } from 'svelte/store' export function computeSecretUrl(secretUrl: string) { - return `${window.location.origin}${base}/public/${get(workspaceStore)}/${secretUrl}` + return `${window.location.origin}/public/${get(workspaceStore)}/${secretUrl}` } diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte index a4bed4affb..1d3d46fc35 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/EmptyInlineScript.svelte @@ -15,6 +15,7 @@ import WorkspaceScriptList from '../settingsPanel/mainInput/WorkspaceScriptList.svelte' import RunnableSelector from '../settingsPanel/mainInput/RunnableSelector.svelte' import { defaultScripts } from '$lib/stores' + import { useOperatorBuilderApps } from '$lib/operatorWriteRights' import DefaultScripts from '$lib/components/DefaultScripts.svelte' import type { Preview } from '$lib/gen' import type { InlineScript } from '../../sharedTypes' @@ -38,6 +39,7 @@ let picker: Drawer | undefined = $state(undefined) const dispatch = createEventDispatcher() + const operatorBuilderApps = useOperatorBuilderApps() async function inferInlineScriptSchema( language: Preview['language'], @@ -104,7 +106,9 @@ - + {#if !$operatorBuilderApps} + + {/if}
@@ -123,21 +127,25 @@
-
Choose a language
+
+ {$operatorBuilderApps ? 'Choose a script or flow' : 'Choose a language'} +
{#if showScriptPicker} {/if} - + {#if !$operatorBuilderApps} + + {/if}
-
+ +
Backend
diff --git a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte index 293d83b4ae..45dbe82c14 100644 --- a/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte +++ b/frontend/src/lib/components/apps/editor/inlineScriptsPanel/InlineScriptRunnableByPath.svelte @@ -3,7 +3,6 @@ const bubble = createBubbler() import { Button, Drawer, DrawerContent } from '$lib/components/common' - import { base } from '$lib/base' import FlowGraphViewer from '$lib/components/FlowGraphViewer.svelte' import Skeleton from '$lib/components/common/skeleton/Skeleton.svelte' import FlowModuleScript from '$lib/components/flows/content/FlowModuleScript.svelte' @@ -19,6 +18,7 @@ CtxAppInput } from '../../inputType' import type { AppViewerContext } from '../../types' + import { useOperatorBuilderApps } from '$lib/operatorWriteRights' import { createEventDispatcher } from 'svelte' import { deepEqual } from 'fast-equals' import { computeFields } from './utils' @@ -37,6 +37,7 @@ import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' const operatingWorkspace = useOperatingWorkspace() + const operatorBuilderApps = useOperatorBuilderApps() interface Props { runnable: RunnableByPath @@ -325,7 +326,7 @@ startIcon={{ icon: GitFork }} endIcon={{ icon: ExternalLink }} target="_blank" - href="{base}/flows/add?hub={hubFlowId}" + href="/flows/add?hub={hubFlowId}" > Fork @@ -346,7 +347,7 @@ startIcon={{ icon: Eye }} endIcon={{ icon: ExternalLink }} target="_blank" - href="{base}/flows/get/{runnable.path}?workspace={$operatingWorkspace}" + href="/flows/get/{runnable.path}?workspace={$operatingWorkspace}" > Details @@ -362,16 +363,18 @@ > Edit - + {#if !$operatorBuilderApps} + + {/if} {/if} > + } +} + +/** Carries `status` like the generated client's `ApiError`, which callers branch on. */ +export class PublicApiError extends Error { + constructor(public status: number) { + super(`Request failed with status ${status}`) + } +} + +/** Prefetched responses answer for the credential the page loaded with; call this when it + * changes (a sign-in or logout on the page) so later reads go to the network. */ +export function dropPrefetched(): void { + delete window.__wmPublicAppPrefetch +} + +function takePrefetched(url: string): Promise | undefined { + const prefetched = window.__wmPublicAppPrefetch + const res = prefetched?.[url] + if (res) delete prefetched![url] + return res +} + +/** GET an API path with the page credential: the bearer token in `OpenAPI.TOKEN` when the + * share link carries one, the session cookie otherwise. A prefetched response is only + * taken in the cookie case, which is the only credential `app.html` sends. */ +export async function publicGet(path: string): Promise { + const url = `${OpenAPI.BASE}${path}` + const token = typeof OpenAPI.TOKEN === 'string' && OpenAPI.TOKEN ? OpenAPI.TOKEN : undefined + const res = await ((!token && takePrefetched(url)) || + fetch(url, { + // As the generated client does: the opaque viewer frame is cross-origin to the + // API, whose `Access-Control-Allow-Origin: *` refuses a credentialed request. + credentials: OpenAPI.WITH_CREDENTIALS ? OpenAPI.CREDENTIALS : undefined, + headers: token ? { Authorization: `Bearer ${token}` } : {} + })) + if (!res.ok) throw new PublicApiError(res.status) + return (res.headers.get('content-type')?.includes('json') ? res.json() : res.text()) as T +} + +export async function getUserExtOrUndefined(workspace: string): Promise { + try { + return mapUserToUserExt( + await publicGet(`/w/${encodeURI(workspace)}/users/whoami`), + workspace + ) + } catch { + return undefined + } +} + +/** Only fades the "Powered by" badge, so a failure is not worth surfacing. */ +export async function getLicenseOrUndefined(): Promise { + try { + return (await publicGet('/ee_license')) || undefined + } catch { + return undefined + } +} + +let appCss: Promise | undefined +/** The app-wide stylesheet, which these routes need only off the raw-app path: sign-in, + * errors, skeletons and low-code apps. A raw app's bundle brings its own styles. */ +export function loadAppCss(): Promise { + return (appCss ??= import('$lib/assets/app.css')) +} diff --git a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte index 08b8ac6219..44f2adb9ac 100644 --- a/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte +++ b/frontend/src/lib/components/apps/editor/settingsPanel/mainInput/RunnableSelector.svelte @@ -12,8 +12,10 @@ import { emptySchema } from '$lib/utils' import { loadSchema } from '$lib/infer' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' + import { useOperatorBuilderApps } from '$lib/operatorWriteRights' const operatingWorkspace = useOperatingWorkspace() + const operatorBuilderApps = useOperatorBuilderApps() import { buildPathRunnableSelection } from './runnableSelectorUtils' type TabType = 'hubscripts' | 'hubflows' | 'workspacescripts' | 'workspaceflows' | 'inlinescripts' @@ -163,10 +165,12 @@ {/if} - {#if !onlyFlow} - + {#if !$operatorBuilderApps} + {#if !onlyFlow} + + {/if} + {/if} -
diff --git a/frontend/src/lib/components/apps/types.ts b/frontend/src/lib/components/apps/types.ts index 75de2d7faf..a72d54507e 100644 --- a/frontend/src/lib/components/apps/types.ts +++ b/frontend/src/lib/components/apps/types.ts @@ -188,6 +188,8 @@ export interface AppEditorProps { // with its author and time. `version` and `head` differ when another deploy landed // beside this one, which is when the out-of-date prompt has something to say. onDeploy?: (e: { version?: number; head?: number; headBy?: string; headAt?: string }) => void + /** True for the route's own editor: its top bar becomes the page header. */ + ownsPageHeader?: boolean } export type App = { diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte new file mode 100644 index 0000000000..49a222148b --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDataSourceMenu.svelte @@ -0,0 +1,150 @@ + + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte new file mode 100644 index 0000000000..2a54fe11d4 --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamMenu.svelte @@ -0,0 +1,73 @@ + + + + + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte new file mode 100644 index 0000000000..8ebd24455c --- /dev/null +++ b/frontend/src/lib/components/assets/AssetGraph/AddDownstreamPill.svelte @@ -0,0 +1,23 @@ + + + + + + + Add downstream step + + diff --git a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte index 880abb685a..57a31e34d8 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AddNode.svelte @@ -1,145 +1,36 @@ - - {#snippet trigger()} - - - {/snippet} - + +
+ + {#snippet trigger()} + + + {/snippet} + +
diff --git a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte index 27f6ce6be7..87b085593c 100644 --- a/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte +++ b/frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte @@ -1,8 +1,11 @@ -
+
onNodePointerEnter(node.id)} + onnodepointerleave={onNodePointerLeave} + onedgepointerenter={onEdgeEnter} + onedgepointerleave={onEdgeLeave} --background-color={false} >
- + + {#if assetsOnlyToggle} + + { + assetsOnly = !e.detail + if (assetsOnlyFolder != undefined) setAssetsOnlyFolder(assetsOnlyFolder, assetsOnly) + }} + /> + {#if withoutPassive.dropped > 0} + { + showPassiveReads = e.detail + try { + localStorage.setItem(SHOW_PASSIVE_READS_KEY, String(e.detail)) + } catch {} + }} + /> + {/if} + + {/if} {#if showMinimap} + +{/if} + +{#if scheduleWizard} + +{/if} + +{#if onAddPipelineScript} + +{/if} + + - + { @@ -2467,7 +2577,12 @@ addEventListener('message', function (e) { {onTakeLatest} {draftBaseVersion} {onRestore} - {onSavedNewAppPath} + onSavedNewAppPath={(newAppPath) => { + if (opWorkspace && $userStore?.email) { + movePreviewSdkConsent($userStore.email, opWorkspace, path, newAppPath) + } + onSavedNewAppPath?.(newAppPath) + }} {policy} {diffDrawer} {newApp} @@ -2484,6 +2599,7 @@ addEventListener('message', function (e) { {getBundle} {onNavigate} {onDeploy} + {onDetails} {onResetToDeployed} {loadedFromDraft} {othersDraftsCount} @@ -2496,6 +2612,7 @@ addEventListener('message', function (e) { sidebarCollapsed={sidebarCollapsed.val} onToggleSidebar={() => (sidebarCollapsed.val = !sidebarCollapsed.val)} {condensedHeader} + {ownsPageHeader} /> {/key} {/if} - {#if buildError} + {#if sdkPrompt} + +
+ {#if sdkPrompt.declined} +
+

+ The preview is blocked until you grant the permissions this app declares. +

+ +
+ {:else} + sdkPrompt && (sdkPrompt.declined = true)} + /> + {/if} +
+ {:else if buildError}
{#if workers} - {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage, native_mode }} + {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage, native_mode, draining }} {@const isWorkerAlive = isWorkerMaybeAlive(last_ping)} {@const tagMismatchInfo = getTagMismatchInfo( custom_tags, @@ -1087,6 +1107,15 @@ {#if native_mode} Native {/if} + {#if draining} + + Draining + {#snippet text()} + This worker received a shutdown signal. It no longer pulls jobs + and exits once its current job is done. + {/snippet} + + {/if} {#if !config || !config.worker_tags || config.worker_tags.length === 0} diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index f055e92ca3..58f9d5d897 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -3,7 +3,7 @@ import { page } from '$app/stores' import { isCloudHosted } from '$lib/cloud' import CenteredPage from '$lib/components/CenteredPage.svelte' - import { Alert, Button, CopyButton, Section, Skeleton, Tab, Tabs } from '$lib/components/common' + import { Alert, Button, Section, Skeleton, Tab, Tabs } from '$lib/components/common' import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' @@ -13,7 +13,7 @@ import ErrorOrRecoveryHandler, { handlerFullPath } from '$lib/components/ErrorOrRecoveryHandler.svelte' - import PageHeader from '$lib/components/PageHeader.svelte' + import PageHeaderContent from '$lib/components/PageHeaderContent.svelte' import ScriptPicker from '$lib/components/ScriptPicker.svelte' import Tooltip from '$lib/components/Tooltip.svelte' @@ -59,7 +59,6 @@ convertBackendSettingsToFrontendSettings, type S3ResourceSettings } from '$lib/workspace_settings' - import { base } from '$lib/base' import ConnectionSection from '$lib/components/ConnectionSection.svelte' import AISettings from '$lib/components/workspaceSettings/AISettings.svelte' import SharedUiSettings from '$lib/components/workspaceSettings/SharedUiSettings.svelte' @@ -827,7 +826,7 @@ }) // Redirect to OAuth flow - window.location.href = `${base}/api/oauth/connect_slack?workspace=${$workspaceStore}` + window.location.href = `/api/oauth/connect_slack?workspace=${$workspaceStore}` } catch (e) { sendUserToast('Failed to save Slack OAuth configuration', true) console.error(e) @@ -1134,10 +1133,14 @@ } // Strip keys from extraArgs that are auto-managed by child components: - // - 'slack': computed by ErrorOrRecoveryHandler's $effect based on handler type + // - 'slack' while the Slack built-in is selected: set by ErrorOrRecoveryHandler's $effect // - 'channel_name': display metadata stripped by SchemaForm's removeExtraKey() - function normalizeHandlerExtraArgs(args: Record): Record { - const { slack: _, channel_name: __, ...rest } = args + function normalizeHandlerExtraArgs( + args: Record, + handlerSelected: ErrorHandler + ): Record { + const { channel_name: _, ...rest } = args + if (handlerSelected === 'slack') delete rest.slack return rest } @@ -1147,7 +1150,10 @@ errorHandlerSelected: initialErrorHandlerSelected, errorHandlerScriptPath: initialErrorHandlerScriptPath, errorHandlerItemKind: initialErrorHandlerItemKind, - errorHandlerExtraArgs: normalizeHandlerExtraArgs(initialErrorHandlerExtraArgs), + errorHandlerExtraArgs: normalizeHandlerExtraArgs( + initialErrorHandlerExtraArgs, + initialErrorHandlerSelected + ), errorHandlerMutedOnCancel: initialErrorHandlerMutedOnCancel, errorHandlerMutedOnUserPath: initialErrorHandlerMutedOnUserPath } @@ -1156,7 +1162,7 @@ errorHandlerSelected: errorHandlerSelected, errorHandlerScriptPath: errorHandlerScriptPath, errorHandlerItemKind: errorHandlerItemKind, - errorHandlerExtraArgs: normalizeHandlerExtraArgs(errorHandlerExtraArgs), + errorHandlerExtraArgs: normalizeHandlerExtraArgs(errorHandlerExtraArgs, errorHandlerSelected), errorHandlerMutedOnCancel: errorHandlerMutedOnCancel, errorHandlerMutedOnUserPath: errorHandlerMutedOnUserPath } @@ -1457,26 +1463,31 @@ } ] ) + + // The label of the entry the sidebar highlights, so the breadcrumb and the sidebar name the + // same pane: both read `tab`, which resolves the URL's aliases (`teams`, `deploy_to`, …). + const tabLabel = $derived( + navigationGroups.flatMap((g) => g.items).find((i) => i.id === tab)?.label + ) - + {#if canAdmin || isForkOwner} - - {#snippet titleActions()} - {#if $workspaceStore} - - {/if} - {/snippet} - {#if $superadmin} - - {/if} + +
-
+ +
-
+
{#if !loadedSettings} {:else if tab == 'users'} @@ -1549,7 +1560,7 @@ @@ -1587,7 +1598,7 @@ @@ -1639,9 +1650,9 @@ } }} onSelect={editSlackCommand} - connectHref="{base}/api/oauth/connect_slack" - createScriptHref="{base}/scripts/add?hub=hub%2F28071%2Fslack%2Fexample_of_responding_to_a_slack_command_slack" - createFlowHref="{base}/flows/add?hub=28" + connectHref="/api/oauth/connect_slack" + createScriptHref="/scripts/add?hub=hub%2F28071%2Fslack%2Fexample_of_responding_to_a_slack_command_slack" + createFlowHref="/flows/add?hub=28" documentationLink="https://www.windmill.dev/docs/integrations/slack" onLoadSettings={loadSettings} display_name={slack_team_name} @@ -1726,7 +1737,7 @@ Slack API . Set the redirect URI to:{' '} - {window.location.origin}{base}/oauth/callback_slack + {window.location.origin}/oauth/callback_slack
@@ -1776,8 +1787,8 @@ }} onSelect={editTeamsCommand} connectHref={undefined} - createScriptHref="{base}/scripts/add?hub=hub%2F11591%2Fteams%2FExample%20of%20responding%20to%20a%20Microsoft%20Teams%20command" - createFlowHref="{base}/flows/add?hub=58" + createScriptHref="/scripts/add?hub=hub%2F11591%2Fteams%2FExample%20of%20responding%20to%20a%20Microsoft%20Teams%20command" + createFlowHref="/flows/add?hub=58" documentationLink="https://www.windmill.dev/docs/integrations/teams" onLoadSettings={loadSettings} display_name={teams_team_name} @@ -1830,7 +1841,7 @@ {:else} + {/if} +{/snippet} + diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/checkout/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/checkout/+page.svelte index b25b8db5c8..62c25fc6b3 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/checkout/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/checkout/+page.svelte @@ -31,7 +31,7 @@ - + {#if !success}
diff --git a/frontend/src/routes/(root)/+layout.svelte b/frontend/src/routes/(root)/+layout.svelte index 06d7d570b2..f14f9e1043 100644 --- a/frontend/src/routes/(root)/+layout.svelte +++ b/frontend/src/routes/(root)/+layout.svelte @@ -1,5 +1,6 @@ + +{@render children?.()} diff --git a/frontend/src/routes/app_embed/[workspace]/[...path]/+page.svelte b/frontend/src/routes/(styled)/app_embed/[workspace]/[...path]/+page.svelte similarity index 81% rename from frontend/src/routes/app_embed/[workspace]/[...path]/+page.svelte rename to frontend/src/routes/(styled)/app_embed/[workspace]/[...path]/+page.svelte index ee83869ee8..9b3a9c1e41 100644 --- a/frontend/src/routes/app_embed/[workspace]/[...path]/+page.svelte +++ b/frontend/src/routes/(styled)/app_embed/[workspace]/[...path]/+page.svelte @@ -78,23 +78,29 @@ } - { - refresh = requestTokenRefresh - loadApp() - }} -> - {#snippet viewer()} - loadApp()} - > - {/snippet} - + +
+ { + refresh = requestTokenRefresh + loadApp() + }} + > + {#snippet viewer()} + loadApp()} + > + {/snippet} + +
diff --git a/frontend/src/routes/approve/[workspace]/[job]/+page.svelte b/frontend/src/routes/(styled)/approve/[workspace]/[job]/+page.svelte similarity index 99% rename from frontend/src/routes/approve/[workspace]/[job]/+page.svelte rename to frontend/src/routes/(styled)/approve/[workspace]/[job]/+page.svelte index e2b6b88c3f..a31fe76670 100644 --- a/frontend/src/routes/approve/[workspace]/[job]/+page.svelte +++ b/frontend/src/routes/(styled)/approve/[workspace]/[job]/+page.svelte @@ -1,6 +1,5 @@ + - { - refresh = requestTokenRefresh - loadApp() - }} -> - {#snippet viewer()} - loadApp()} - > - {/snippet} - + +
+ { + refresh = requestTokenRefresh + loadApp() + }} + > + {#snippet viewer()} + loadApp()} + > + {/snippet} + +
diff --git a/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte b/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte index b2a29b6316..fb32a8ac9b 100644 --- a/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte +++ b/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte @@ -1,18 +1,22 @@ - { - refresh = requestTokenRefresh - loadApp() - }} -> - {#snippet viewer()} - loadApp()} - > - {/snippet} - + +
+ { + refresh = requestTokenRefresh + loadApp() + }} + > + {#snippet viewer()} + loadApp()} + > + {/snippet} + +
diff --git a/frontend/svelte.config.js b/frontend/svelte.config.js index 15634551e6..64b39b5d7d 100644 --- a/frontend/svelte.config.js +++ b/frontend/svelte.config.js @@ -35,9 +35,6 @@ const config = { // revisions (SvelteKit only full-page reloads on a missing chunk if this changed). version: { name: process.env.WM_BUILD_VERSION || pkg.version }, prerender: { entries: [] }, - paths: { - base: process.env.VITE_BASE_URL ?? '' - }, alias: { $system_prompts: '../system_prompts/auto-generated', $oauth_connect_registry: '../backend/oauth_connect.json', diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 5ae1e390e3..a07034d2c1 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -108,25 +108,34 @@ const isClientBuild = (bundle) => Object.keys(bundle).some((file) => file.startsWith('_app/immutable/')) /** - * Fail the build if a public app URL statically loads the low-code runtime or monaco. + * Fail the build if a public app URL statically loads more than a raw app needs. * * These pages also serve raw apps, which only need a small shell around their bundle's - * iframe. One static import of AppPreview (or of anything reaching monaco) makes every - * public app page preload ~650 chunks instead of ~55. See loadAppPreview.ts. + * iframe, and the page shows nothing until its static imports have loaded. One static + * import of AppPreview (or of anything reaching monaco) makes every public app page + * preload ~650 chunks; one of the generated client, `$lib/stores` or `$lib/utils` drags + * in most of the app shell. The root layout wraps these pages, so it must not import + * app.css either. See loadAppPreview.ts and publicAppApi.ts. */ function assertLeanPublicAppRoutes() { // Route directories, so +page.js counts as well as +page.svelte. const routes = ['/src/routes/public/[workspace]/[...secret]/', '/src/routes/a/[...path]/'] const forbidden = [ '/src/lib/components/apps/editor/AppPreview.svelte', - '/node_modules/monaco-editor/' + '/node_modules/monaco-editor/', + '/src/lib/assets/app.css', + '/src/lib/gen/services.gen.ts', + '/src/lib/stores.ts', + '/src/lib/utils.ts', + '/src/lib/components/Login.svelte', + '/node_modules/@melt-ui/' ] return { name: 'wm-assert-lean-public-app-routes', generateBundle(_options, bundle) { if (!isClientBuild(bundle)) return - for (const route of routes) { - assertStaticClosureExcludes(this, bundle, route, forbidden, 'loadAppPreview.ts') + for (const route of [...routes, '/src/routes/+layout.svelte']) { + assertStaticClosureExcludes(this, bundle, route, forbidden, 'publicAppApi.ts') } } } @@ -372,7 +381,19 @@ const config = { // which makes the *chunk* graph cyclic where the module graph is not, and // modules then evaluate against uninitialized bindings. See // docs/frontend-import-cycles.md. - advancedChunks: { groups: [{ name: 'gen', test: /[\\/]src[\\/]lib[\\/]gen[\\/]/ }] } + // The client's config is split out for the public app routes, which set the + // credential without loading the client. It imports nothing at runtime, so + // its chunk cannot be part of a cycle. + advancedChunks: { + groups: [ + { + name: 'gen-config', + test: /[\\/]src[\\/]lib[\\/]gen[\\/]core[\\/]OpenAPI\.ts$/, + priority: 1 + }, + { name: 'gen', test: /[\\/]src[\\/]lib[\\/]gen[\\/]/ } + ] + } } } }, diff --git a/integration_tests/test/identity_script_test.py b/integration_tests/test/identity_script_test.py index dc56c4956c..638919fb22 100644 --- a/integration_tests/test/identity_script_test.py +++ b/integration_tests/test/identity_script_test.py @@ -14,11 +14,6 @@ echo ${x} export async function main(x: number) { return x; } -""", - "deno": """ -export async function main(x: number) { - return x; -} """, "go": """ package inner @@ -72,11 +67,6 @@ class TestIdentityScript(unittest.TestCase): result = self._client.run_sync(path, {"x": 5}) self.assertEqual(result, 5) - def test_deno(self): - path = PATH_TEMPLATE.format(lang="deno") - result = self._client.run_sync(path, {"x": 5}) - self.assertEqual(result, 5) - def test_go(self): path = PATH_TEMPLATE.format(lang="go") result = self._client.run_sync(path, {"x": 5}) diff --git a/integration_tests/test/increment_flow_test.py b/integration_tests/test/increment_flow_test.py index c4682cb8b3..2a2f5b88d9 100644 --- a/integration_tests/test/increment_flow_test.py +++ b/integration_tests/test/increment_flow_test.py @@ -38,10 +38,10 @@ FLOW_VALUE = """ "value": { "tag": "", - "lock": "{\\n \\"version\\": \\"3\\",\\n \\"remote\\": {}\\n}\\n", + "lock": "{\\n \\"dependencies\\": {}\\n}\\n//bun.lockb\\n", "type": "rawscript", "content": "export async function main(x: number) {\\n return x + 1\\n}\\n", - "language": "deno", + "language": "bun", "input_transforms": { "x": diff --git a/integration_tests/test/windmill_sdk_test.py b/integration_tests/test/windmill_sdk_test.py index 6b5dcefbe9..11a4fa7886 100644 --- a/integration_tests/test/windmill_sdk_test.py +++ b/integration_tests/test/windmill_sdk_test.py @@ -8,15 +8,6 @@ from .wmill_integration_test_utils import WindmillClient VARIABLE_PATH = "u/admin/test_variable" VARIABLE_VALUE = "Hello world!" -DENO_SCRIPT_PATH = "u/admin/deno_sdk_test_script" -DENO_SCRIPT_VALUE = """ -import * as wmill from "npm:windmill-client@{version}" -export async function main() {{ - const val = await wmill.getVariable('u/admin/test_variable') - return val; -}} -""" - BUN_SCRIPT_PATH = "u/admin/bun_sdk_test_script" BUN_SCRIPT_VALUE = """ import * as wmill from "windmill-client@{version}" @@ -63,11 +54,6 @@ class TestWindmillSdk(unittest.TestCase): path=VARIABLE_PATH, value=VARIABLE_VALUE, ) - cls._client.create_script( - path=DENO_SCRIPT_PATH, - content=DENO_SCRIPT_VALUE.format(version=cls._dev_version), - language="deno", - ) # TODO: See skipped annotations below # cls._client.create_script( # path=BUN_SCRIPT_PATH, @@ -89,7 +75,6 @@ class TestWindmillSdk(unittest.TestCase): def tearDownClass(cls) -> None: if cls._running_latest: cls._client.set_npm_config_registry("") - cls._client.delete_script(path=DENO_SCRIPT_PATH) # TODO: See skipped annotations below # cls._client.delete_script(path=BUN_SCRIPT_PATH) # cls._client.delete_script(path=PYTHON_SCRIPT_PATH) @@ -97,13 +82,6 @@ class TestWindmillSdk(unittest.TestCase): cls._client.delete_variable(path=VARIABLE_PATH) cls._client.set_npm_config_registry("") - @unittest.skipUnless( - os.environ.get("WMILL_RUNNING_DEV", "0") == "1", "Runs on dev version only" - ) - def test_deno_sdk_usable(self): - result = self._client.run_sync(DENO_SCRIPT_PATH, {}) - self.assertEqual(result, VARIABLE_VALUE) - @unittest.skipUnless( # os.environ.get("WMILL_RUNNING_DEV", "0") == "1", "Runs on dev version only" False, diff --git a/lsp/Pipfile b/lsp/Pipfile index c4bb35dca2..4aee9bdf9f 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.826.0" +wmill = ">=1.830.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/lsp/pyls_launcher.py b/lsp/pyls_launcher.py index 9e522d8ce5..b3fbe7fd9a 100644 --- a/lsp/pyls_launcher.py +++ b/lsp/pyls_launcher.py @@ -1,6 +1,9 @@ +import asyncio import logging +import shutil import subprocess import threading +import time import os import urllib.request import urllib.error @@ -149,8 +152,221 @@ class LanguageServerWebSocketHandler(websocket.WebSocketHandler): class PyrightLS(LanguageServerWebSocketHandler): procargs = ["pipenv", "run", "pyright-langserver", "--stdio"] -class DiagnosticLS(LanguageServerWebSocketHandler): - procargs = ["diagnostic-languageserver", "--stdio", "--log-level", "4"] +SHELLCHECK_BIN = shutil.which("shellcheck") +# The linter is configured here rather than by the editor's initializationOptions. +SHELLCHECK_ARGS = ["--norc", "--format=json1", "-"] +SHELLCHECK_TIMEOUT_SECS = float(os.environ.get("SHELLCHECK_TIMEOUT_SECS", "10")) +SHELLCHECK_MAX_DOC_CHARS = int(os.environ.get("SHELLCHECK_MAX_DOC_CHARS", "200000")) +SHELLCHECK_MAX_DOCS_PER_CONN = 8 +# A document whose checks finish faster than this is checked on every change +# with no pause; see ShellcheckLS._changed. +SHELLCHECK_INSTANT_SECS = 0.05 +SHELLCHECK_MAX_DEBOUNCE_SECS = 1.0 +# Shared by every connection: shellcheck is CPU-bound and single-threaded, so +# more concurrent runs than cores only makes all of them slower. +_shellcheck_slots = asyncio.Semaphore(os.cpu_count() or 2) + +_SHELLCHECK_SEVERITIES = {"error": 1, "warning": 2, "info": 3, "style": 3} + + +def _utf16_col(line, col): + """shellcheck counts code points, LSP positions count UTF-16 code units.""" + col = max(col - 1, 0) + if line.isascii(): + return col + return len(line[:col].encode("utf-16-le")) // 2 + + +def shellcheck_diagnostics(text, output): + lines = text.split("\n") + + def position(line, col): + line = max(line - 1, 0) + return { + "line": line, + "character": _utf16_col(lines[line], col) if line < len(lines) else 0, + } + + diagnostics = [] + for c in json.loads(output).get("comments", []): + code = "SC%d" % c["code"] + diagnostics.append( + { + "range": { + "start": position(c["line"], c["column"]), + "end": position(c["endLine"], c["endColumn"]), + }, + "severity": _SHELLCHECK_SEVERITIES.get(c["level"], 3), + "code": code, + "codeDescription": {"href": "https://www.shellcheck.net/wiki/" + code}, + "source": "shellcheck", + "message": c["message"], + } + ) + return diagnostics + + +class ShellcheckLS(websocket.WebSocketHandler): + """Lints shell documents by running shellcheck itself, speaking only the + slice of LSP that takes: document sync in, diagnostics out.""" + + def open(self): + self.docs = {} + self.tasks = {} + self.cost = {} + self.linted = {} + + def check_origin(self, origin): + return True + + def on_message(self, message): + try: + msg = json.loads(message) + method = msg.get("method") + params = msg.get("params") or {} + if method == "initialize": + self._send( + { + "jsonrpc": "2.0", + "id": msg.get("id"), + "result": { + "capabilities": { + "textDocumentSync": {"openClose": True, "change": 1} + }, + "serverInfo": {"name": "windmill-shellcheck"}, + }, + } + ) + elif method == "textDocument/didOpen": + doc = params["textDocument"] + self._changed(doc["uri"], doc.get("version"), doc["text"]) + elif method == "textDocument/didChange": + doc = params["textDocument"] + self._changed( + doc["uri"], doc.get("version"), params["contentChanges"][-1]["text"] + ) + elif method == "textDocument/didClose": + self._forget(params["textDocument"]["uri"]) + elif method == "shutdown": + self._send({"jsonrpc": "2.0", "id": msg.get("id"), "result": None}) + elif method is not None and "id" in msg: + self._send( + { + "jsonrpc": "2.0", + "id": msg["id"], + "error": {"code": -32601, "message": "Method not found"}, + } + ) + except (ValueError, KeyError, IndexError, TypeError, AttributeError): + log.warning("shellcheck: dropping malformed message") + + def on_close(self): + for uri in list(self.docs): + self._forget(uri) + + def _send(self, msg): + try: + self.write_message(json.dumps(msg)) + except websocket.WebSocketClosedError: + pass + + def _forget(self, uri): + task = self.tasks.pop(uri, None) + if task: + task.cancel() + self.docs.pop(uri, None) + self.cost.pop(uri, None) + self.linted.pop(uri, None) + + def _changed(self, uri, version, text): + if not isinstance(uri, str) or not isinstance(text, str): + raise TypeError + if uri not in self.docs and len(self.docs) >= SHELLCHECK_MAX_DOCS_PER_CONN: + return + if len(text) > SHELLCHECK_MAX_DOC_CHARS: + # Not checked, so not kept either: reported as having no findings. + text = None + self.docs[uri] = (version, text) + if uri in self.linted and self.linted[uri] == text: + return + running = self.tasks.pop(uri, None) + # The pause before a check is what that document's checks cost: one that + # is cheap is checked at once, one that is not waits out a typing burst + # rather than burning a core on results the next keystroke discards. + delay = self.cost.get(uri, 0) + if delay < SHELLCHECK_INSTANT_SECS: + delay = 0 + if running and not running.done(): + running.cancel() + delay = max(delay, SHELLCHECK_INSTANT_SECS) + self.tasks[uri] = asyncio.ensure_future(self._lint(uri, version, text, delay)) + + async def _lint(self, uri, version, text, delay): + if delay: + await asyncio.sleep(delay) + diagnostics = [] + if SHELLCHECK_BIN and text is not None: + async with _shellcheck_slots: + started = time.monotonic() + try: + output = await self._run(text) + self.cost[uri] = 0 + finally: + # A killed check counts too, as a lower bound: it is the only + # measure a document edited faster than it can be checked gets. + if uri in self.docs: + self.cost[uri] = min( + max(time.monotonic() - started, self.cost.get(uri, 0)), + SHELLCHECK_MAX_DEBOUNCE_SECS, + ) + if output is None: + return + try: + diagnostics = shellcheck_diagnostics(text, output) + except (ValueError, KeyError, TypeError) as e: + log.warning("shellcheck: unreadable output: %s", e) + return + if self.docs.get(uri) != (version, text): + return + self.linted[uri] = text + params = {"uri": uri, "diagnostics": diagnostics} + if version is not None: + params["version"] = version + self._send( + { + "jsonrpc": "2.0", + "method": "textDocument/publishDiagnostics", + "params": params, + } + ) + + async def _run(self, text): + try: + proc = await asyncio.create_subprocess_exec( + SHELLCHECK_BIN, + *SHELLCHECK_ARGS, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + except OSError as e: + log.warning("shellcheck: could not start: %s", e) + return None + try: + output, _ = await asyncio.wait_for( + proc.communicate(text.encode("utf-8", "replace")), + SHELLCHECK_TIMEOUT_SECS, + ) + return output + except asyncio.TimeoutError: + log.warning("shellcheck: timed out on a %d char document", len(text)) + return None + finally: + # Also reached when a newer change cancels this check mid-run. + if proc.returncode is None: + proc.kill() + await proc.wait() + class RuffLS(LanguageServerWebSocketHandler): procargs = ["ruff", "server"] @@ -201,11 +417,14 @@ if __name__ == "__main__": # spawned `ruff server` picks it up. start_ruff_config_poller() + if not SHELLCHECK_BIN: + log.warning("shellcheck is not on PATH: shell scripts will not be linted") + port = int(os.environ.get("PORT", "3001")) app = web.Application( [ (r"/ws/pyright", PyrightLS), - (r"/ws/diagnostic", DiagnosticLS), + (r"/ws/diagnostic", ShellcheckLS), (r"/ws/ruff", RuffLS), (r"/ws/deno", DenoLS), (r"/ws/go", GoLS), diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index b3c6faacf4..e3759ee6a6 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.826.0 + version: 1.830.0 title: OpenFlow Spec contact: name: Ruben Fiszel @@ -145,7 +145,7 @@ components: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: "Retry with exponential backoff: the n-th retry waits multiplier × seconds^n" properties: attempts: type: integer @@ -156,7 +156,7 @@ components: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index efec616858..8385126639 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.826.0' + ModuleVersion = '1.830.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 3aa64f361c..411bc09cc8 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.826.0" +version = "1.830.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill/tests/test_workflow.py b/python-client/wmill/tests/test_workflow.py index ceed39bfeb..d32ae1458b 100644 --- a/python-client/wmill/tests/test_workflow.py +++ b/python-client/wmill/tests/test_workflow.py @@ -7,7 +7,7 @@ import pytest from datetime import datetime, timezone -from wmill.client import WorkflowCtx, _StepSuspend, TaskError, workflow, task, step, sleep, parallel, wait_for_approval, _run_workflow, _run_workflow_async +from wmill.client import WorkflowCtx, _StepSuspend, TaskError, workflow, task, task_script, step, sleep, parallel, wait_for_approval, _run_workflow, _run_workflow_async class _FakeItems(dict): @@ -752,6 +752,84 @@ class TestChildModeParallel: assert result["result"] == 8 +class TestSdkCallsInsideTaskBody: + """A task body runs as one child job, so the SDK calls it makes cannot be + dispatched as steps of their own.""" + + def _run_child(self, wf, key): + async def bounded(): + return await asyncio.wait_for( + _run_workflow_async(wf, {"completed_steps": {}, "_executing_key": key}, {}), + timeout=2, + ) + + return asyncio.run(bounded()) + + def test_nested_task_and_step_run_in_the_tasks_own_job(self): + @task + async def inner(x: str): + return f"inner:{x}" + + @task + def sync_leaf(x: str): + return f"sync:{x}" + + # A sync body calls its tasks without awaiting, whatever called it. + @task + def sync_inner(x: str): + return sync_leaf(x).upper().lower() + + @task + async def outer(x: str): + via_task = await inner(x) + via_sync = await sync_inner(x) + via_step = await step("s", lambda: "stepped") + return f"{via_task}+{via_sync}+{via_step}" + + @workflow + async def wf(): + return await outer("a") + + result = self._run_child(wf, "outer") + assert result == {"type": "complete", "result": "inner:a+sync:a+stepped"} + + def test_sibling_task_of_the_workflow_body_is_not_run(self): + # The workflow body keeps replaying beside the task body, and its own + # calls must stay parked: running them would execute a sibling task in + # this job. + ran = [] + + @task + async def inner(x: str): + ran.append(x) + return x + + @task + async def outer(x: str): + await asyncio.sleep(0) + return await inner("nested") + + @workflow + async def wf(): + return await asyncio.gather(outer("a"), inner("sibling")) + + result = self._run_child(wf, "outer") + assert result == {"type": "complete", "result": "nested"} + assert ran == ["nested"] + + def test_step_that_would_need_a_dispatch_fails_instead_of_hanging(self): + @task + async def dispatching(): + return await task_script("f/other/script")() + + @workflow + async def wf(): + return await dispatching() + + with pytest.raises(RuntimeError, match="cannot be called from inside a task"): + self._run_child(wf, "dispatching") + + class TestKeyDeterminism: def test_keys_consistent_across_replays(self): r1 = _run_workflow(det_wf, {}, {"n": 3}) diff --git a/python-client/wmill/wmill/client.py b/python-client/wmill/wmill/client.py index 81e5b34e16..824db9741f 100644 --- a/python-client/wmill/wmill/client.py +++ b/python-client/wmill/wmill/client.py @@ -196,12 +196,15 @@ class Windmill: args: dict = None, scheduled_in_secs: int = None, tag: str = None, + retry: dict = None, ) -> str: """Internal helper for running scripts asynchronously.""" args = args or {} params = {"scheduled_in_secs": scheduled_in_secs} if scheduled_in_secs else {} if tag: params["tag"] = tag + if retry: + params["retry"] = json.dumps(retry) if os.environ.get("WM_JOB_ID"): params["parent_job"] = os.environ.get("WM_JOB_ID") if os.environ.get("WM_ROOT_FLOW_JOB_ID"): @@ -222,9 +225,15 @@ class Windmill: args: dict = None, scheduled_in_secs: int = None, tag: str = None, + retry: dict = None, ) -> str: - """Create a script job by path and return its job id.""" - return self._run_script_async_internal(path=path, args=args, scheduled_in_secs=scheduled_in_secs, tag=tag) + """Create a script job by path and return its job id. + + ``retry`` re-runs the job when it fails, with the policy a schedule takes, + e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is + the first attempt's; each retry is a job of its own, recorded as a child of it. + """ + return self._run_script_async_internal(path=path, args=args, scheduled_in_secs=scheduled_in_secs, tag=tag, retry=retry) def run_script_by_hash_async( self, @@ -232,9 +241,10 @@ class Windmill: args: dict = None, scheduled_in_secs: int = None, tag: str = None, + retry: dict = None, ) -> str: - """Create a script job by hash and return its job id.""" - return self._run_script_async_internal(hash_=hash_, args=args, scheduled_in_secs=scheduled_in_secs, tag=tag) + """Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``.""" + return self._run_script_async_internal(hash_=hash_, args=args, scheduled_in_secs=scheduled_in_secs, tag=tag, retry=retry) def run_flow_async( self, @@ -813,6 +823,22 @@ class Windmill: else: return r.json() + def worker_is_draining(self, job_id: Optional[str] = None) -> bool: + """Whether the worker running the job has received its shutdown signal. + + A draining worker waits for its current job to finish before exiting. A long-running + script can poll this and return early so the worker can be replaced. + + Args: + job_id: Job ID (defaults to current WM_JOB_ID) + + Returns: + True if the worker running the job is draining + """ + job_id = job_id or os.environ.get("WM_JOB_ID") + + return self.get(f"/w/{self.workspace}/jobs/worker_is_draining/{job_id}").json() + def set_flow_user_state(self, key: str, value: Any) -> None: """Set the user state of a flow at a given key""" flow_id = self.get_root_job_id() @@ -1618,6 +1644,7 @@ def run_script_by_path_async( args: Dict[str, Any] = None, scheduled_in_secs: Union[None, int] = None, tag: str = None, + retry: Dict[str, Any] = None, ) -> str: """Create a script job by path and return its job ID. @@ -1626,6 +1653,8 @@ def run_script_by_path_async( args: Script arguments scheduled_in_secs: Delay before execution in seconds tag: Override the worker tag the job runs on + retry: Re-run the job when it fails, with the policy a schedule takes, + e.g. ``{"constant": {"attempts": 3, "seconds": 10}}`` Returns: Job ID string @@ -1635,6 +1664,7 @@ def run_script_by_path_async( args=args, scheduled_in_secs=scheduled_in_secs, tag=tag, + retry=retry, ) @@ -1644,6 +1674,7 @@ def run_script_by_hash_async( args: Dict[str, Any] = None, scheduled_in_secs: Union[None, int] = None, tag: str = None, + retry: Dict[str, Any] = None, ) -> str: """Create a script job by hash and return its job ID. @@ -1652,6 +1683,7 @@ def run_script_by_hash_async( args: Script arguments scheduled_in_secs: Delay before execution in seconds tag: Override the worker tag the job runs on + retry: Re-run the job when it fails, see ``run_script_by_path_async`` Returns: Job ID string @@ -1661,6 +1693,7 @@ def run_script_by_hash_async( args=args, scheduled_in_secs=scheduled_in_secs, tag=tag, + retry=retry, ) @@ -2010,6 +2043,17 @@ def get_progress(job_id: Optional[str] = None) -> Any: return _client.get_progress(job_id) +@init_global_client +def worker_is_draining(job_id: Optional[str] = None) -> bool: + """ + Whether the worker running this job has received its shutdown signal. + + A draining worker waits for its current job to finish before exiting. A long-running + script can poll this and return early so the worker can be replaced. + """ + return _client.worker_is_draining(job_id) + + def set_shared_state_pickle(value: Any, path="state.pickle") -> None: """ Set the state in the shared folder using pickle @@ -2722,6 +2766,7 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]]: import asyncio as _asyncio import contextvars as _contextvars +import inspect as _inspect import sys as _sys import traceback as _traceback @@ -2949,6 +2994,64 @@ _workflow_ctx: _contextvars.ContextVar["WorkflowCtx"] = _contextvars.ContextVar( "_workflow_ctx" ) +# Set while a task body runs in its child job: ``"async"`` or ``"sync"`` after +# the kind of function the body is. A flag on the ctx cannot stand in: under +# ``asyncio.gather`` the workflow body keeps replaying beside the task body, and +# its calls must still park rather than run a sibling task's body in this job. +_in_task_body: _contextvars.ContextVar[Optional[str]] = _contextvars.ContextVar( + "_in_task_body", default=None +) + + +def _dispatch_ctx() -> "WorkflowCtx | None": + """The ctx SDK calls dispatch through, or None where they run in place: + outside a workflow, and inside a task body, which is one job and has no + rounds of its own to dispatch or checkpoint a step in.""" + if _in_task_body.get(): + return None + return _workflow_ctx.get(None) + + +async def _resolved(value): + return value + + +def _call_in_task_body(func, caller: str, args, kwargs): + """Run a task called from inside a task body in place, round-tripping its + result as a dispatched one would be. + + The workflow body awaits every task, sync ones included, so an async caller + gets an awaitable back either way; a sync caller gets the value. The scope + is re-marked for the nested body, which may be of the other kind.""" + + async def run_async(coro): + token = _in_task_body.set("async") + try: + return _json_round_trip(await coro) + finally: + _in_task_body.reset(token) + + if _inspect.iscoroutinefunction(func): + return run_async(func(*args, **kwargs)) + token = _in_task_body.set("sync") + try: + result = func(*args, **kwargs) + finally: + _in_task_body.reset(token) + if _asyncio.iscoroutine(result): + return run_async(result) + result = _json_round_trip(result) + return _resolved(result) if caller == "async" else result + + +def _assert_not_inside_task_body(what: str) -> None: + if _in_task_body.get(): + raise RuntimeError( + f"{what} cannot be called from inside a task: a task runs as a single " + "job and cannot suspend or dispatch steps. Call it from the @workflow " + "body instead." + ) + class WorkflowCtx: """Internal context for workflow replay/suspension. @@ -3073,15 +3176,20 @@ class WorkflowCtx: raise _StepSuspend({"mode": "sleep", "key": key, "seconds": seconds, "steps": []}) async def _resolved(self, value): - return value + return await _resolved(value) async def _execute_directly(self, func, **kwargs): + token = _in_task_body.set( + "async" if _inspect.iscoroutinefunction(func) else "sync" + ) try: result = func(**kwargs) if _asyncio.iscoroutine(result): result = await result except Exception as exc: raise _StepFailure(exc) from exc + finally: + _in_task_body.reset(token) raise _StepSuspend({"mode": "step_complete", "steps": [], "result": result}) async def _never_resolve(self): @@ -3323,7 +3431,9 @@ def task( - **v2 (inside @workflow)**: dispatches as a checkpoint step. - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. - - **Standalone**: executes the function body directly. + - **Standalone, or called from inside another v2 task**: executes the + function body directly. A task runs as a single job, so the tasks it + calls run in that same job, without their own options. A task runs as its own job, so its result is always encoded as JSON and decoded back before the caller sees it: a ``datetime`` comes back as a @@ -3403,12 +3513,16 @@ def task( @functools.wraps(func) def wrapper(*args, **kwargs): # WAC v2: inside a @workflow context - ctx = _workflow_ctx.get(None) + ctx = _dispatch_ctx() if ctx is not None: script = task_path if task_path else task_name merged = _merge_args(args, kwargs) return ctx._next_step(task_name, script, func, _task_options=_task_opts, **merged) + caller = _in_task_body.get() + if caller is not None: + return _call_in_task_body(func, caller, args, kwargs) + # WAC v1: running inside a Windmill job but not in a @workflow if ( os.environ.get("WM_JOB_ID") is not None @@ -3487,6 +3601,7 @@ def task_script( _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None def wrapper(**kwargs): + _assert_not_inside_task_body(f'task_script("{path}")') ctx = _workflow_ctx.get(None) if ctx is not None: return ctx._next_step(name, path, dispatch_type="script", _task_options=_opts, **kwargs) @@ -3526,6 +3641,7 @@ def task_flow( _opts = {k: v for k, v in {"timeout": timeout, "tag": tag, "cache_ttl": cache_ttl, "priority": priority, "concurrent_limit": concurrency_limit, "concurrency_key": concurrency_key, "concurrency_time_window_s": concurrency_time_window_s, "retry": _checked_retry(retry)}.items() if v is not None} or None def wrapper(**kwargs): + _assert_not_inside_task_body(f'task_flow("{path}")') ctx = _workflow_ctx.get(None) if ctx is not None: return ctx._next_step(name, path, dispatch_type="flow", _task_options=_opts, **kwargs) @@ -3560,8 +3676,11 @@ async def step(name: str, fn): ``fn``'s result is encoded as JSON and decoded back before it is returned, so the round that runs the body sees the same types every replay sees: a ``datetime`` comes back as a string, a tuple as a list. + + Called from inside a task body, ``fn`` runs in place and nothing is + checkpointed: a task is a single job and is never replayed. """ - ctx: WorkflowCtx | None = _workflow_ctx.get(None) + ctx = _dispatch_ctx() if ctx is not None: return await ctx._run_inline_step(name, fn) result = fn() @@ -3576,9 +3695,9 @@ async def sleep(seconds: int): """Server-side sleep — suspend the workflow for the given duration without holding a worker. Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. - Outside a workflow, falls back to ``asyncio.sleep``. + Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. """ - ctx: WorkflowCtx | None = _workflow_ctx.get(None) + ctx = _dispatch_ctx() if ctx is not None: return await ctx._sleep(seconds) await _asyncio.sleep(seconds) @@ -3616,6 +3735,7 @@ async def wait_for_approval( await step("notify", lambda: send_email(urls["resume"], urls["cancel"])) result = await wait_for_approval(key="manager", timeout=3600) """ + _assert_not_inside_task_body("wait_for_approval()") ctx: WorkflowCtx | None = _workflow_ctx.get(None) if ctx is not None: return await ctx._wait_for_approval( diff --git a/rust-client/src/client.rs b/rust-client/src/client.rs index 451dc58d85..bf47e7e95a 100644 --- a/rust-client/src/client.rs +++ b/rust-client/src/client.rs @@ -785,6 +785,7 @@ impl Windmill { None, None, None, + None, ) .await? } else { @@ -808,6 +809,7 @@ impl Windmill { None, None, None, + None, ) .await? }; diff --git a/system_prompts/README.md b/system_prompts/README.md index cc16473c6c..a651c48692 100644 --- a/system_prompts/README.md +++ b/system_prompts/README.md @@ -46,6 +46,7 @@ a CLI-only workflow file (`flow-cli.md`, `script-cli.md`, `raw-app-cli.md`). | resources | `getResourcePrompt()` | `resources` | `resources.md` | | workflow-as-code | `getWorkflowAsCodePrompt(lang)` | `write-workflow-as-code` | `workflow-as-code.md`, the WAC SDK | | pipeline | `getPipelinePrompt()` | `write-pipeline` | `pipeline-base.md` | +| data tables | `getDatatablePrompt()` | `datatables` | `datatable.md` | | triggers, schedules, preview, CLI commands | — | same names | CLI only | The app chat embeds `RAW_APP_BASE` directly (`chat/app/core.ts`). diff --git a/system_prompts/auto-generated/cli/cli-commands.md b/system_prompts/auto-generated/cli/cli-commands.md index 7b13702da6..a1daf47e32 100644 --- a/system_prompts/auto-generated/cli/cli-commands.md +++ b/system_prompts/auto-generated/cli/cli-commands.md @@ -30,7 +30,7 @@ app related commands - `app dev [app_folder:string]` - Start a development server for building apps with live reload and hot module replacement - `--port ` - Port to run the dev server on (will find next available port if occupied) - `--host ` - Host to bind the dev server to - - `--entry ` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) + - `--entry ` - Entry point file (default: index.ts for Svelte, index.tsx otherwise) - `--no-open` - Don't automatically open the browser - `--recording` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development - `app lint [app_folder:string]` - Lint a raw app folder to validate structure and buildability @@ -41,7 +41,7 @@ app related commands - `app new` - create a new raw app from a template - `--summary ` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode. - `--path ` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - - `--framework ` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. + - `--framework ` - Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode. - `--datatable ` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - `--schema ` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - `--overwrite` - Overwrite the target directory if it already exists, without prompting. @@ -727,6 +727,8 @@ user related commands **Subcommands:** +- `user list` - List the users of the workspace + - `--json` - Output as JSON (for piping to jq) - `user add [password:string]` - Create a user - `--superadmin` - Specify to make the new user superadmin. - `--company ` - Specify to set the company of the new user. diff --git a/system_prompts/auto-generated/flow.md b/system_prompts/auto-generated/flow.md index 7a965a17d8..e5dd862a3b 100644 --- a/system_prompts/auto-generated/flow.md +++ b/system_prompts/auto-generated/flow.md @@ -284,6 +284,10 @@ can call one, not an agent used as a tool. - `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope - `flow_input.iter.value` - Current iteration value inside a `forloopflow`; in a `whileloopflow` it is just the iteration index (a plain number, same as `flow_input.iter.index`) - `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`) +- After a suspend step (approval), the next step's javascript expressions also get: + - `resume` - the last resume payload (the resume form values); with `continue_on_disapprove_timeout`, a disapproval or timeout yields `{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }` instead + - `resumes` - every resume payload, in arrival order + - `approvers` - a plain array of strings, one per resume: for a resume through the `resume`/`cancel` URL from `getResumeUrls(approver)`, the `approver` passed there if any; otherwise (including the `approvalPage` link) the resuming user's username, else `"anonymous"`, e.g. `approvers[0]`. It is NOT the `{ resume_id, approver }` objects stored in the flow status, so `approvers[0].approver` is always undefined ## Loop Structure Rules @@ -594,4 +598,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\nstate with calibrated probabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\nstate with calibrated probabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} \ No newline at end of file diff --git a/system_prompts/auto-generated/index.d.ts b/system_prompts/auto-generated/index.d.ts index d77f699a33..8ba83f7f74 100644 --- a/system_prompts/auto-generated/index.d.ts +++ b/system_prompts/auto-generated/index.d.ts @@ -4,5 +4,6 @@ export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(language?: string): string; export declare function getPipelinePrompt(): string; +export declare function getDatatablePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; diff --git a/system_prompts/auto-generated/index.ts b/system_prompts/auto-generated/index.ts index ade95efe9e..60bb79b031 100644 --- a/system_prompts/auto-generated/index.ts +++ b/system_prompts/auto-generated/index.ts @@ -73,6 +73,13 @@ export function getPipelinePrompt(): string { ].filter(Boolean).join('\n\n'); } +// Helper for data table schema changes, migrations and permissions (chat consumers) +export function getDatatablePrompt(): string { + return [ + prompts.DATATABLE_BASE + ].filter(Boolean).join('\n\n'); +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { diff --git a/system_prompts/auto-generated/prompts.d.ts b/system_prompts/auto-generated/prompts.d.ts index e74555504e..cfdde52495 100644 --- a/system_prompts/auto-generated/prompts.d.ts +++ b/system_prompts/auto-generated/prompts.d.ts @@ -5,6 +5,7 @@ export declare const FLOW_BASE: string; export declare const RESOURCES_BASE: string; export declare const RAW_APP_BASE: string; export declare const PIPELINE_BASE: string; +export declare const DATATABLE_BASE: string; export declare const WORKFLOW_AS_CODE_BASE: string; export declare const FLOW_CHAT_SPECIAL_MODULES: string; export declare const SDK_TYPESCRIPT: string; diff --git a/system_prompts/auto-generated/prompts.ts b/system_prompts/auto-generated/prompts.ts index 8f6a2cc2c1..54d869f727 100644 --- a/system_prompts/auto-generated/prompts.ts +++ b/system_prompts/auto-generated/prompts.ts @@ -311,6 +311,10 @@ can call one, not an agent used as a tool. - \`results.step_id.property\` - Access specific property from a previous step output only when that step result is in scope - \`flow_input.iter.value\` - Current iteration value inside a \`forloopflow\`; in a \`whileloopflow\` it is just the iteration index (a plain number, same as \`flow_input.iter.index\`) - \`flow_input.iter.index\` - Current loop index when inside a loop (\`forloopflow\` or \`whileloopflow\`) +- After a suspend step (approval), the next step's javascript expressions also get: + - \`resume\` - the last resume payload (the resume form values); with \`continue_on_disapprove_timeout\`, a disapproval or timeout yields \`{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }\` instead + - \`resumes\` - every resume payload, in arrival order + - \`approvers\` - a plain array of strings, one per resume: for a resume through the \`resume\`/\`cancel\` URL from \`getResumeUrls(approver)\`, the \`approver\` passed there if any; otherwise (including the \`approvalPage\` link) the resuming user's username, else \`"anonymous"\`, e.g. \`approvers[0]\`. It is NOT the \`{ resume_id, approver }\` objects stored in the flow status, so \`approvers[0].approver\` is always undefined ## Loop Structure Rules @@ -886,7 +890,7 @@ def main(db: postgresql): export const RAW_APP_BASE = `# Windmill Raw Apps -Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. +Raw apps let you build custom frontends with React or Svelte that connect to Windmill backend runnables and datatables. ## App shape @@ -902,7 +906,7 @@ A new app gets the setup Windmill's own new-app dialog gives one: a framework, t ### Framework -Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18 and Svelte 5 are for when they are asked for. These three are the only frameworks the raw app bundler builds. When the user asks to build or convert an app in another framework, write no code in it: say it is not available and offer these three instead. ### Data setup @@ -935,7 +939,7 @@ An app the user just started from the new-app dialog is already set up: its file ### Entrypoint -The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte and Vue. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` / \`App.vue\` and keep the entrypoint as the mount shim. +The entrypoint is \`index.tsx\` for React and \`index.ts\` for Svelte. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty \`
\` and auto-renders nothing, so the entrypoint must mount a top-level \`App\` itself. Keep the UI in \`App.tsx\` / \`App.svelte\` and keep the entrypoint as the mount shim. React (\`index.tsx\`): @@ -947,7 +951,7 @@ import App from './App' createRoot(document.getElementById('root')!).render() \`\`\` -Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. Vue (\`index.ts\`): \`createApp(App).mount('#root')\`. +Svelte (\`index.ts\`): \`mount(App, { target: document.getElementById('root')! })\`. **Never replace the entrypoint with a bare component** (\`export default function App() { ... }\` and no mount call). A component that is defined but never mounted renders a blank screen with **no error thrown** — it never executes, so nothing reaches the console or the error overlay. If an app renders blank, check that the entrypoint still mounts \`App\` into \`#root\`. @@ -1020,16 +1024,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`windmill-chat\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add \`@windmill-labs/chat-sdk\` to \`package.json\` and use it directly, it detects the app's Windmill and credential. \`\`\`tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() \`\`\` -\`windmill-chat/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`windmill-chat/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. +\`@windmill-labs/chat-sdk/ai-sdk\` gives a \`ChatTransport\` for the Vercel AI SDK's \`useChat\`, \`@windmill-labs/chat-sdk/assistant-ui\` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs \`jobs:run\` in its frontend SDK scopes, plus \`flow_conversations:write\` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -1112,6 +1116,10 @@ A draft app is reachable by nobody; deploying is what exposes it, and its backen Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. When a deploy widens who may open the app, tell the user in plain words. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are \`jobs:run\`, \`jobs:read\`, \`users:read\`, \`resources:read\`, \`variables:read\`, \`flow_conversations:read\` and \`flow_conversations:write\`; grant only the ones the frontend calls. + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the \`wmill\` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. @@ -1123,6 +1131,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer 3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. 4. **Qualify table names**: an unqualified name means the \`public\` schema, so write every other table as \`schema.table\`, in table creation and queries alike. The app's \`data\` config sets the default datatable and schema its tables go in; use them consistently across runnables. 5. **Pass the role**: when the app's \`data.roles\` gives a data table a role, every \`wmill.datatable\` call on it passes that role (\`wmill.datatable('main', { role: 'analyst' })\` in TypeScript, \`wmill.datatable('main', role='analyst')\` in Python). The role only reaches what it was granted, so a query outside it fails with \`permission denied\`. +6. **Per-viewer rows**: a data table with no entry in \`data.roles\` is called with no role. Unless its default role is \`admin\`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as \`pg_has_role(current_user, '', 'member')\` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no \`data.roles\` entry. ### Querying in TypeScript (Bun/Deno) @@ -1174,7 +1183,7 @@ def main(user_id: str): 7. **Mark sensitive UI with \`data-wm-no-record\`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for \`backendAsync\` + \`waitJob\`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use \`windmill-chat\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use \`@windmill-labs/chat-sdk\` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. `; export const PIPELINE_BASE = `# Data pipeline authoring @@ -1186,10 +1195,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **\`duckdb\`** node that materializes its output into a **DuckLake** table with \`-- materialize ducklake:///
{scriptHash} @@ -150,7 +149,7 @@ {jobId.substring(24)}
\` (in a DuckDB node the annotation uses SQL \`--\` comment syntax; write the body as a bare \`SELECT\` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - \`postgresql\` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- \`bun\` / \`python3\` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- \`bun\` / \`python3\` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with \`wmill.ducklake()\`, or into a data table with \`wmill.datatable()\`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +\`\`\`python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +\`\`\` + +\`\`\`ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake\`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(\${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)\`.fetch() +} +\`\`\` + +- Use \`INSERT INTO
SELECT ...\` instead of \`CREATE OR REPLACE\` to append (an incremental pull), with a one-time \`CREATE TABLE IF NOT EXISTS\` first. +- Assign the client to a variable (\`lake = wmill.ducklake()\`) and keep the SQL a **string literal** (Python) or a template without \`sql.raw\` (TS): that is what lets the graph record the node's write to \`ducklake://main/
\`. A chained \`wmill.ducklake().query(...)\` or SQL built in a variable records no output, so the downstream edge never forms. +- \`wmill.ducklake("")\` targets a non-default catalog; \`wmill.datatable("")\` writes a Postgres data table the same way (Python \`db.query(sql, *args)\` with \`$1\` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: \`-- on ducklake://main/stripe_charges\`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and \`s3://\` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -1232,7 +1279,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the output via the SDK (\`wmill.writeS3File(...)\`, ducklake helpers, …) and let it be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. +> **A MANAGED \`// materialize\` is DuckDB-only**, and its target must be a DuckLake table (\`ducklake:///
\`). Deploy **rejects** a \`ducklake://\` \`// materialize\` on any other language (\`python3\`, \`bun\`, \`postgresql\`). For a non-DuckDB node writing the lake, do **not** use \`// materialize\` — write the table itself with \`wmill.ducklake()\` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use \`duckdb\` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: \`// materialize manual dbt:////\`, where \`\` is a warehouse the workspace configures under Settings → dbt. \`manual\` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql \`CREATE TABLE\` / \`INSERT\`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a \`source\`: the declared relation and the dbt model land on ONE graph node, and a downstream \`// on dbt:////\` fires when the ingestion node completes. @@ -1291,6 +1338,47 @@ FROM ducklake.main.orders GROUP BY 1 \`\`\` `; +export const DATATABLE_BASE = `# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, \`main\` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an \`up\` and an optional \`down\`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in \`BEGIN; ... END;\` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as \`CREATE INDEX CONCURRENTLY\`: such a migration can fail half applied. +- Give it a \`down\` that undoes the \`up\` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (\`CREATE SCHEMA IF NOT EXISTS\`), and point what you are building at it. A raw app names its schema in its \`data\` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. \`admin\` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (\`wmill.datatable('main', { role: 'analyst' })\`, \`role='analyst'\` in Python, \`datatable://main?role=analyst\`) and holds that role alone, or names none and is resolved from the data table's default role; +- \`permission denied\` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to \`admin\` to get past it; +- a migration runs as the role its SQL declares on its first line, \`-- role \`, the same one in its \`down\`. One that declares none runs as \`admin\`, which only those who may use \`admin\` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. +`; + export const WORKFLOW_AS_CODE_BASE = `# Windmill Workflow-as-Code Writing Guide ## Scope @@ -1385,6 +1473,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with \`@task()\` or \`@task\`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or \`step()\` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. \`sleep()\` there waits in the job, and \`taskScript()\` / \`taskFlow()\` / \`waitForApproval()\` (\`task_script()\` / \`task_flow()\` / \`wait_for_approval()\`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: \`\`\`typescript @@ -1648,21 +1738,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * \`{ scheduledInSeconds?, tag?, retry? }\`, e.g. \`{ retry: { constant: { attempts: 3, seconds: 10 } } }\` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1718,6 +1810,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -1977,6 +2077,9 @@ async sleep(seconds: number): Promise * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -2210,10 +2313,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# \`\`retry\`\` re-runs the job when it fails, with the policy a schedule takes, +# e.g. \`\`{"constant": {"attempts": 3, "seconds": 10}}\`\`. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See \`\`run_script_by_path_async\`\` for \`\`retry\`\`. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -2386,6 +2493,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -2803,7 +2922,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -2889,12 +3010,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -3016,8 +3140,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a \`workflow()\`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a \`Date\` comes back as a string, a @@ -3061,6 +3186,9 @@ export function workflow(fn: (...args: any[]) => Promise) * \`fn\`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a \`Date\` * comes back as a string, a \`Map\` as \`{}\`. {@link Jsonified} is that shape. + * + * Called from inside a task body, \`fn\` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> @@ -3159,7 +3287,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a \`\`datetime\`\` comes back as a @@ -3245,12 +3375,15 @@ def workflow(func) # \`\`fn\`\`'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a \`\`datetime\`\` comes back as a string, a tuple as a list. +# +# Called from inside a task body, \`\`fn\`\` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after \`\`seconds\`\`. -# Outside a workflow, falls back to \`\`asyncio.sleep\`\`. +# Outside a workflow, or inside a task body, falls back to \`\`asyncio.sleep\`\`. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. @@ -3475,7 +3608,7 @@ class SqlQuery: export const OPENFLOW_SCHEMA = `## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \\u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \\u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \\u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\\n\`memory_id\`). Without a memory id the agent runs without memory.\\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the \`memory_id\` here.\\nThe step's own \`memory_id\` is not read while this kind is set; switch the kind to \`window\`\\nto use it. Without a \`context_length\`, or with 0, it is \`off\` and reads \`previous_messages\`.\\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's \`memory_id\`), replacing\\nits older part with a summary as the conversation approaches the model's context window.\\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\\nWindmill does not know needs one; those fall back to 128000.\\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to \`off\` with \`previous_messages\` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see \`memory_id\`. While it is off, a step can supply its history in \`previous_messages\`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\\nValid values: 'text' (default) - plain text response, 'image' - image generation\\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\\nflow.input syntax. Required unless memory is off and \`previous_messages\` supplies\\nthe prompt; image output always needs it.\\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\\nwas started with (the chat conversation, an app chat session or the \`memory_id\` run\\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\\nacross every run; an expression such as \`flow_input.customer_id\` keeps one memory per\\nkey. When it evaluates to an empty value the agent runs without memory. Read only\\nwhile \`memory\` is \`window\` or \`compaction\`: it is ignored when memory is off, and an\\nolder \`auto\` or \`manual\` memory reads neither history input.\\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\\nand the user message. Read only while \`memory\` is off or absent: managed memory\\nignores it, and an older \`auto\` or \`manual\` memory reads neither history input.\\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\\nFormat: Array<{ bucket: string, key: string }> - S3 object references\\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in \`tools\` the agent may call\\nthis run. Leaving it unset carries every one of them; an empty array carries none.\\nA tool is named as the model is shown it. An entry the model is shown nothing of is\\nnamed by what identifies it instead: an MCP server by its resource path, carrying\\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\\n(no tool may take that name).\\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\\nRange: 0.0 to 2.0 (provider-dependent)\\n- 0.0 = deterministic, focused responses\\n- 0.7 = balanced (common default)\\n- 1.0+ = more creative/random\\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\\nRange: 1-1000.\\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default \`flow\`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable \`ai_agent\` resource (hybrid linking). When set, the agent brain\\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\\nthat resource; the module's input_transforms then only carry the flow-local inputs\\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\\nshared resource; overlaid onto the tools' input_transforms at runtime \\u2014 including when\\n\`agent\` is unset, since a step forked for editing keeps these overrides until it is saved\\nback or unlinked.\\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\\nstate with calibrated probabilities, instead of generating text.\\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\\ncheap and its answers are structured. Its result is { output: { : answer },\\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\\nthe answers, follow it with a branchone whose branch exprs read\\nresults..output..choice. Usable as an AI agent tool, where the calling agent\\nsupplies the state.\\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\\nAn object with descriptive keys holding only what the questions need works best.\\nExample: { message: flow_input.message, plan: results.a.plan }\\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\\n- 'choice': criteria maps each option to its description (up to 255); the answer\\n has choice, probabilities and confidence\\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\\n has score, legend, probabilities and confidence\\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\\n noul, the probability of yes (0 to 1)\\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default \`flow\`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}}`; export const CLI_COMMANDS = `# Windmill CLI Commands @@ -3509,7 +3642,7 @@ app related commands - \`app dev [app_folder:string]\` - Start a development server for building apps with live reload and hot module replacement - \`--port \` - Port to run the dev server on (will find next available port if occupied) - \`--host \` - Host to bind the dev server to - - \`--entry \` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) + - \`--entry \` - Entry point file (default: index.ts for Svelte, index.tsx otherwise) - \`--no-open\` - Don't automatically open the browser - \`--recording\` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development - \`app lint [app_folder:string]\` - Lint a raw app folder to validate structure and buildability @@ -3520,7 +3653,7 @@ app related commands - \`app new\` - create a new raw app from a template - \`--summary \` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode. - \`--path \` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - - \`--framework \` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. + - \`--framework \` - Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode. - \`--datatable \` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - \`--schema \` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - \`--overwrite\` - Overwrite the target directory if it already exists, without prompting. @@ -4206,6 +4339,8 @@ user related commands **Subcommands:** +- \`user list\` - List the users of the workspace + - \`--json\` - Output as JSON (for piping to jq) - \`user add [password:string]\` - Create a user - \`--superadmin\` - Specify to make the new user superadmin. - \`--company \` - Specify to set the company of the new user. @@ -5726,6 +5861,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; \`\`\` +## Array parameters + +Declare a list with a \`[]\` suffix on the element type (\`(varchar[])\`, +\`(int[])\`), or \`(array)\` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +\`PARSE_JSON(?)\`, and cast to \`ARRAY\` where a function expects one. + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +\`\`\` + +Or join on the elements with \`FLATTEN\`: + +\`\`\`sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +\`\`\` + ## Receiving an S3Object as a script parameter Declare the arg with type \`(s3object)\`. Windmill renders an S3 file picker for diff --git a/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml b/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml index 70a955d16d..5d5b0f8a0d 100644 --- a/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/amqp_trigger.schema.yaml @@ -84,7 +84,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -95,7 +96,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml b/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml index d21d006e8d..e57c92ebc3 100644 --- a/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/azure_trigger.schema.yaml @@ -71,7 +71,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -82,7 +83,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/email_trigger.schema.yaml b/system_prompts/auto-generated/schemas/email_trigger.schema.yaml index 1d4e81ac2c..79786fc8db 100644 --- a/system_prompts/auto-generated/schemas/email_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/email_trigger.schema.yaml @@ -54,7 +54,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -65,7 +66,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml b/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml index f35b33ad74..6755d6545c 100644 --- a/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/gcp_trigger.schema.yaml @@ -94,7 +94,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -105,7 +106,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/http_trigger.schema.yaml b/system_prompts/auto-generated/schemas/http_trigger.schema.yaml index f484366783..b428fbf390 100644 --- a/system_prompts/auto-generated/schemas/http_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/http_trigger.schema.yaml @@ -133,7 +133,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -144,7 +145,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml b/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml index 09217692e5..6549ce1b10 100644 --- a/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/kafka_trigger.schema.yaml @@ -135,7 +135,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -146,7 +147,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml b/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml index 2801cc9a73..f167e2965f 100644 --- a/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/mqtt_trigger.schema.yaml @@ -96,7 +96,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -107,7 +108,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml b/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml index 239d568fcc..e7620683e6 100644 --- a/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/nats_trigger.schema.yaml @@ -70,7 +70,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -81,7 +82,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml b/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml index be196a7bc3..79cca28595 100644 --- a/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/postgres_trigger.schema.yaml @@ -63,7 +63,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -74,7 +75,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/schedule.schema.yaml b/system_prompts/auto-generated/schemas/schedule.schema.yaml index d0ea85d6fe..7097be7d25 100644 --- a/system_prompts/auto-generated/schemas/schedule.schema.yaml +++ b/system_prompts/auto-generated/schemas/schedule.schema.yaml @@ -69,7 +69,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -80,7 +81,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml b/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml index 4f41c32421..32d6400973 100644 --- a/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/sqs_trigger.schema.yaml @@ -69,7 +69,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -80,7 +81,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml b/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml index 780b0f2245..8184d3815a 100644 --- a/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml +++ b/system_prompts/auto-generated/schemas/websocket_trigger.schema.yaml @@ -172,7 +172,8 @@ properties: description: Seconds to wait between retries exponential: type: object - description: Retry with exponential backoff (delay doubles each time) + description: 'Retry with exponential backoff: the n-th retry waits multiplier + × seconds^n' properties: attempts: type: integer @@ -183,7 +184,8 @@ properties: seconds: type: integer minimum: 1 - description: Initial delay in seconds + description: Base of the exponential (seconds); the n-th retry waits multiplier + × seconds^n, where n counts constant retries too random_factor: type: integer minimum: 0 diff --git a/system_prompts/auto-generated/script.md b/system_prompts/auto-generated/script.md index 0517cf1897..d178ddb6c0 100644 --- a/system_prompts/auto-generated/script.md +++ b/system_prompts/auto-generated/script.md @@ -1409,6 +1409,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Array parameters + +Declare a list with a `[]` suffix on the element type (`(varchar[])`, +`(int[])`), or `(array)` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +`PARSE_JSON(?)`, and cast to `ARRAY` where a function expects one. + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +``` + +Or join on the elements with `FLATTEN`: + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +``` + ## Receiving an S3Object as a script parameter Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for @@ -1567,21 +1588,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -1637,6 +1660,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -1896,6 +1927,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> @@ -2129,10 +2163,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -2305,6 +2343,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -2722,7 +2772,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -2808,12 +2860,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/sdks/python.md b/system_prompts/auto-generated/sdks/python.md index 97db78b3d4..1964c88685 100644 --- a/system_prompts/auto-generated/sdks/python.md +++ b/system_prompts/auto-generated/sdks/python.md @@ -60,10 +60,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -236,6 +240,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -653,7 +669,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -739,12 +757,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/sdks/typescript.md b/system_prompts/auto-generated/sdks/typescript.md index 55f59b1f36..da9e79ec94 100644 --- a/system_prompts/auto-generated/sdks/typescript.md +++ b/system_prompts/auto-generated/sdks/typescript.md @@ -124,21 +124,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -194,6 +196,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -453,6 +463,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/sdks/wac-python.md b/system_prompts/auto-generated/sdks/wac-python.md index a8e98a9d4f..6dd307dd24 100644 --- a/system_prompts/auto-generated/sdks/wac-python.md +++ b/system_prompts/auto-generated/sdks/wac-python.md @@ -36,7 +36,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -122,12 +124,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/sdks/wac-typescript.md b/system_prompts/auto-generated/sdks/wac-typescript.md index 608d75d18c..dd04a64de0 100644 --- a/system_prompts/auto-generated/sdks/wac-typescript.md +++ b/system_prompts/auto-generated/sdks/wac-typescript.md @@ -67,8 +67,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -112,6 +113,9 @@ export function workflow(fn: (...args: any[]) => Promise) * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/cli-commands/SKILL.md b/system_prompts/auto-generated/skills/cli-commands/SKILL.md index 48ea832a31..95bbbc657b 100644 --- a/system_prompts/auto-generated/skills/cli-commands/SKILL.md +++ b/system_prompts/auto-generated/skills/cli-commands/SKILL.md @@ -35,7 +35,7 @@ app related commands - `app dev [app_folder:string]` - Start a development server for building apps with live reload and hot module replacement - `--port ` - Port to run the dev server on (will find next available port if occupied) - `--host ` - Host to bind the dev server to - - `--entry ` - Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise) + - `--entry ` - Entry point file (default: index.ts for Svelte, index.tsx otherwise) - `--no-open` - Don't automatically open the browser - `--recording` - Frame the app in a shell with a Record button, to capture a replayable session recording of the app under development - `app lint [app_folder:string]` - Lint a raw app folder to validate structure and buildability @@ -46,7 +46,7 @@ app related commands - `app new` - create a new raw app from a template - `--summary ` - App summary (short description). Skips the prompt when provided. Triggers non-interactive mode. - `--path ` - App path (e.g., f/folder/my_app or u/username/my_app). Skips the prompt when provided. Triggers non-interactive mode. - - `--framework ` - Framework template: react19 | react18 | svelte5 | vue. Skips the prompt when provided. Triggers non-interactive mode. + - `--framework ` - Framework template: react19 | react18 | svelte5. Skips the prompt when provided. Triggers non-interactive mode. - `--datatable ` - Datatable to wire up. Without this flag in non-interactive mode, no datatable is configured. - `--schema ` - Schema to use with --datatable. If it doesn't exist yet, a sql_to_apply/ migration (CREATE SCHEMA IF NOT EXISTS) is written for it. - `--overwrite` - Overwrite the target directory if it already exists, without prompting. @@ -732,6 +732,8 @@ user related commands **Subcommands:** +- `user list` - List the users of the workspace + - `--json` - Output as JSON (for piping to jq) - `user add [password:string]` - Create a user - `--superadmin` - Specify to make the new user superadmin. - `--company ` - Specify to set the company of the new user. diff --git a/system_prompts/auto-generated/skills/datatables/SKILL.md b/system_prompts/auto-generated/skills/datatables/SKILL.md new file mode 100644 index 0000000000..1c1cb9eede --- /dev/null +++ b/system_prompts/auto-generated/skills/datatables/SKILL.md @@ -0,0 +1,46 @@ +--- +name: datatables +description: MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions. +--- + +# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, `main` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an `up` and an optional `down`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in `BEGIN; ... END;` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as `CREATE INDEX CONCURRENTLY`: such a migration can fail half applied. +- Give it a `down` that undoes the `up` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + +`wmill datatable migrate new ` scaffolds `migrations/datatable//_.up.sql` and `.down.sql`, `wmill datatable migrate up` pushes the local files and applies what is pending, `wmill datatable migrate down` rolls back the most recent one and `wmill datatable migrate status` lists them. All take `-d ` and default to `main`. + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (`CREATE SCHEMA IF NOT EXISTS`), and point what you are building at it. A raw app names its schema in its `data` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. `admin` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (`wmill.datatable('main', { role: 'analyst' })`, `role='analyst'` in Python, `datatable://main?role=analyst`) and holds that role alone, or names none and is resolved from the data table's default role; +- `permission denied` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to `admin` to get past it; +- a migration runs as the role its SQL declares on its first line, `-- role `, the same one in its `down`. One that declares none runs as `admin`, which only those who may use `admin` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. diff --git a/system_prompts/auto-generated/skills/raw-app/SKILL.md b/system_prompts/auto-generated/skills/raw-app/SKILL.md index 2f7cd52d5f..bd21352995 100644 --- a/system_prompts/auto-generated/skills/raw-app/SKILL.md +++ b/system_prompts/auto-generated/skills/raw-app/SKILL.md @@ -17,7 +17,7 @@ You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. `f/folder/my_app` or `u/username/my_app` -3. **framework** — one of `react19` (recommended), `react18`, `svelte5`, `vue` +3. **framework** — one of `react19` (recommended), `react18`, `svelte5` **summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. @@ -89,7 +89,7 @@ my_app__raw_app/ ├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) ├── raw_app.yaml # App configuration (summary, path, data settings) ├── index.tsx # Frontend entry point -├── App.tsx # Main React/Svelte/Vue component +├── App.tsx # Main React/Svelte component ├── index.css # Styles ├── package.json # Frontend dependencies ├── wmill.ts # Auto-generated backend type definitions (DO NOT EDIT) @@ -188,7 +188,7 @@ data: - `/
` — Specific table in public schema - `/:
` — Table in specific schema -**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role. +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role, if it has one, plus the roles of whoever is using the app unless that default is `admin`. ## SQL Migrations (sql_to_apply/) @@ -247,7 +247,7 @@ For the rest, tell the user which command fits their intent and let them run it # Windmill Raw Apps -Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. +Raw apps let you build custom frontends with React or Svelte that connect to Windmill backend runnables and datatables. ## App shape @@ -263,7 +263,7 @@ A new app gets the setup Windmill's own new-app dialog gives one: a framework, t ### Framework -Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18 and Svelte 5 are for when they are asked for. These three are the only frameworks the raw app bundler builds. When the user asks to build or convert an app in another framework, write no code in it: say it is not available and offer these three instead. ### Data setup @@ -293,7 +293,7 @@ Then build what was chosen: the data table and schema become the app's `data.dat ### Entrypoint -The entrypoint is `index.tsx` for React and `index.ts` for Svelte and Vue. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty `
` and auto-renders nothing, so the entrypoint must mount a top-level `App` itself. Keep the UI in `App.tsx` / `App.svelte` / `App.vue` and keep the entrypoint as the mount shim. +The entrypoint is `index.tsx` for React and `index.ts` for Svelte. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty `
` and auto-renders nothing, so the entrypoint must mount a top-level `App` itself. Keep the UI in `App.tsx` / `App.svelte` and keep the entrypoint as the mount shim. React (`index.tsx`): @@ -305,7 +305,7 @@ import App from './App' createRoot(document.getElementById('root')!).render() ``` -Svelte (`index.ts`): `mount(App, { target: document.getElementById('root')! })`. Vue (`index.ts`): `createApp(App).mount('#root')`. +Svelte (`index.ts`): `mount(App, { target: document.getElementById('root')! })`. **Never replace the entrypoint with a bare component** (`export default function App() { ... }` and no mount call). A component that is defined but never mounted renders a blank screen with **no error thrown** — it never executes, so nothing reaches the console or the error overlay. If an app renders blank, check that the entrypoint still mounts `App` into `#root`. @@ -378,16 +378,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `@windmill-labs/chat-sdk` to `package.json` and use it directly, it detects the app's Windmill and credential. ```tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() ``` -`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. +`@windmill-labs/chat-sdk/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `@windmill-labs/chat-sdk/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -471,6 +471,18 @@ A draft app is reachable by nobody; deploying is what exposes it, and its backen Either one lets those people run the app's backend runnables, so never open an app up unless the user asks. `public: true` in `raw_app.yaml` deploys the app for anonymous users, and `guests: true` for guests. Remove the line and the next push closes the app again. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are `jobs:run`, `jobs:read`, `users:read`, `resources:read`, `variables:read`, `flow_conversations:read` and `flow_conversations:write`; grant only the ones the frontend calls. +In `raw_app.yaml`: + +```yaml +sandbox: true +frontend_api: [jobs:run, users:read] +``` + +A file with a `sandbox` key states both settings: `sandbox: false` turns isolation off, and a missing `frontend_api` means no scopes. A file without the key leaves the deployed app's isolation and scopes as they are, so removing `sandbox: true` does not unsandbox it. `frontend_api` needs `sandbox: true`; the push fails otherwise. + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. @@ -482,6 +494,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer 3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. 4. **Qualify table names**: an unqualified name means the `public` schema, so write every other table as `schema.table`, in table creation and queries alike. The app's `data` config sets the default datatable and schema its tables go in; use them consistently across runnables. 5. **Pass the role**: when the app's `data.roles` gives a data table a role, every `wmill.datatable` call on it passes that role (`wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python). The role only reaches what it was granted, so a query outside it fails with `permission denied`. +6. **Per-viewer rows**: a data table with no entry in `data.roles` is called with no role. Unless its default role is `admin`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as `pg_has_role(current_user, '', 'member')` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no `data.roles` entry. `wmill datatable list` lists the workspace's data tables. Create or change tables with a migration in `sql_to_apply/` (see "SQL Migrations" above), then add them to `data.tables` in `raw_app.yaml`. @@ -535,4 +548,4 @@ def main(user_id: str): 7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. diff --git a/system_prompts/auto-generated/skills/write-flow/SKILL.md b/system_prompts/auto-generated/skills/write-flow/SKILL.md index 352fe2ed48..4b0889c3d8 100644 --- a/system_prompts/auto-generated/skills/write-flow/SKILL.md +++ b/system_prompts/auto-generated/skills/write-flow/SKILL.md @@ -371,6 +371,10 @@ can call one, not an agent used as a tool. - `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope - `flow_input.iter.value` - Current iteration value inside a `forloopflow`; in a `whileloopflow` it is just the iteration index (a plain number, same as `flow_input.iter.index`) - `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`) +- After a suspend step (approval), the next step's javascript expressions also get: + - `resume` - the last resume payload (the resume form values); with `continue_on_disapprove_timeout`, a disapproval or timeout yields `{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }` instead + - `resumes` - every resume payload, in arrival order + - `approvers` - a plain array of strings, one per resume: for a resume through the `resume`/`cancel` URL from `getResumeUrls(approver)`, the `approver` passed there if any; otherwise (including the `approvalPage` link) the resuming user's username, else `"anonymous"`, e.g. `approvers[0]`. It is NOT the `{ resume_id, approver }` objects stored in the flow status, so `approvers[0].approver` is always undefined ## Loop Structure Rules @@ -682,4 +686,4 @@ Reference a specific resource using `$res:` prefix: ## OpenFlow Schema -{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff (delay doubles each time)","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Initial delay in seconds"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\nstate with calibrated probabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} +{"OpenFlow":{"type":"object","description":"Top-level flow definition containing metadata, configuration, and the flow structure","properties":{"summary":{"type":"string","description":"Short description of what this flow does"},"description":{"type":"string","description":"Detailed documentation for this flow"},"value":{"$ref":"#/components/schemas/FlowValue"},"schema":{"type":"object","description":"JSON Schema for flow inputs. Use this to define input parameters, their types, defaults, and validation. For resource inputs, set type to 'object' and format to 'resource-' (e.g., 'resource-stripe')"},"on_behalf_of_email":{"type":"string","description":"Address of the account the flow runs on behalf of. Derived from on_behalf_of on read; accepted on write, where it is resolved to the account it names."},"on_behalf_of":{"type":"string","description":"The flow runs with the permissions of this identity: u/{username}, g/{group}, or a bare email when the username is itself email-shaped. The only stored half of the identity; on_behalf_of_email is derived from it. Omit it when writing and it is resolved from that address instead."}},"required":["summary","value"]},"FlowValue":{"type":"object","description":"The flow structure containing modules and optional preprocessor/failure handlers","properties":{"modules":{"type":"array","description":"Array of steps that execute in sequence. Each step can be a script, subflow, loop, or branch","items":{"$ref":"#/components/schemas/FlowModule"}},"failure_module":{"description":"Special module that executes when the flow fails. Receives error object with message, name, stack, and step_id. Must have id 'failure'. Only supports script/rawscript types","$ref":"#/components/schemas/FlowModule"},"preprocessor_module":{"description":"Special module that runs before the first step on external triggers. Must have id 'preprocessor'. Only supports script/rawscript types. Cannot reference other step results","$ref":"#/components/schemas/FlowModule"},"same_worker":{"type":"boolean","description":"If true, all steps run on the same worker for better performance"},"preserve_step_tags":{"type":"boolean","description":"If true and the flow runs on a custom worker tag, steps that declare their own non-empty tag run on it instead of inheriting the flow tag. Steps without their own tag still inherit the flow tag."},"concurrent_limit":{"type":"number","description":"Maximum number of concurrent executions of this flow"},"concurrency_key":{"type":"string","description":"Expression to group concurrent executions (e.g., by user ID)"},"concurrency_time_window_s":{"type":"number","description":"Time window in seconds for concurrent_limit"},"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce flow executions"},"debounce_key":{"type":"string","description":"Expression to group debounced executions"},"debounce_args_to_accumulate":{"type":"array","description":"Arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds that a job can be debounced"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of times a job can be debounced"},"skip_expr":{"type":"string","description":"JavaScript expression to conditionally skip the entire flow"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for flow results"},"cache_ignore_s3_path":{"type":"boolean"},"delete_after_secs":{"type":"integer","description":"If set, delete the flow job's args, result and logs after this many seconds following job completion"},"flow_env":{"type":"object","description":"Environment variables available to all steps. Values can be strings, JSON values, or special references: '$var:path' (workspace variable) or '$res:path' (resource).","additionalProperties":{}},"priority":{"type":"number","description":"Execution priority (higher numbers run first)"},"early_return":{"type":"string","description":"JavaScript expression to return early from the flow"},"chat_input_enabled":{"type":"boolean","description":"Whether this flow accepts chat-style input"},"notes":{"type":"array","description":"Sticky notes attached to the flow","items":{"$ref":"#/components/schemas/FlowNote"}},"groups":{"type":"array","description":"Semantic groups of modules for organizational purposes","items":{"$ref":"#/components/schemas/FlowGroup"}}},"required":["modules"]},"Retry":{"type":"object","description":"Retry configuration for failed module executions","properties":{"constant":{"type":"object","description":"Retry with constant delay between attempts","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"seconds":{"type":"integer","description":"Seconds to wait between retries"}}},"exponential":{"type":"object","description":"Retry with exponential backoff: the n-th retry waits multiplier \u00d7 seconds^n","properties":{"attempts":{"type":"integer","description":"Number of retry attempts"},"multiplier":{"type":"integer","description":"Multiplier for exponential backoff"},"seconds":{"type":"integer","minimum":1,"description":"Base of the exponential (seconds); the n-th retry waits multiplier \u00d7 seconds^n, where n counts constant retries too"},"random_factor":{"type":"integer","minimum":0,"maximum":100,"description":"Random jitter percentage (0-100) to avoid thundering herd"}}},"retry_if":{"$ref":"#/components/schemas/RetryIf"}}},"FlowNote":{"type":"object","description":"A sticky note attached to a flow for documentation and annotation","properties":{"id":{"type":"string","description":"Unique identifier for the note"},"text":{"type":"string","description":"Content of the note"},"position":{"type":"object","description":"Position of the note in the flow editor","properties":{"x":{"type":"number","description":"X coordinate"},"y":{"type":"number","description":"Y coordinate"}},"required":["x","y"]},"size":{"type":"object","description":"Size of the note in the flow editor","properties":{"width":{"type":"number","description":"Width in pixels"},"height":{"type":"number","description":"Height in pixels"}},"required":["width","height"]},"color":{"type":"string","description":"Color of the note, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Any other value renders unstyled."},"type":{"type":"string","enum":["free","group"],"description":"Type of note - 'free' for standalone notes. 'group' notes are deprecated; segment a flow with FlowValue.groups instead."},"locked":{"type":"boolean","default":false,"description":"Whether the note is locked and cannot be edited or moved"},"contained_node_ids":{"type":"array","items":{"type":"string"},"description":"For group notes, the IDs of nodes contained within this group"}},"required":["id","text","color","type"]},"FlowGroup":{"type":"object","description":"A semantic group of flow modules for organizational purposes. Does not affect execution \u2014 modules remain in their original position in the flow. Groups provide naming and collapsibility in the editor. Members are computed dynamically from all nodes on paths between start_id and end_id.","properties":{"summary":{"type":"string","description":"Display name for this group"},"note":{"type":"string","description":"Markdown note shown below the group header"},"autocollapse":{"type":"boolean","default":false,"description":"If true, this group is collapsed by default in the flow editor. UI hint only."},"start_id":{"type":"string","description":"ID of the first flow module in this group (topological entry point)"},"end_id":{"type":"string","description":"ID of the last flow module in this group (topological exit point)"},"color":{"type":"string","description":"Color for the group in the flow editor, one of: yellow, blue, green, purple, pink, orange, red, cyan, lime, gray. Omit it to let the editor pick one."}},"required":["start_id","end_id"]},"RetryIf":{"type":"object","description":"Conditional retry based on error or result","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to retry. Has access to 'result' and 'error' variables"}},"required":["expr"]},"StopAfterIf":{"type":"object","description":"Early termination condition for a module","properties":{"skip_if_stopped":{"type":"boolean","description":"If true, following steps are skipped when this condition triggers"},"expr":{"type":"string","description":"JavaScript expression evaluated after the module runs. Can use 'result' (step's result) or 'flow_input'. Return true to stop"},"error_message":{"type":"string","nullable":true,"description":"Custom error message when stopping with an error. Mutually exclusive with skip_if_stopped. If set to a non-empty string, the flow stops with this error. If empty string, a default error message is used. If null or omitted, no error is raised."},"error_include_result":{"type":"boolean","description":"When stopping with an error (error_message set), embed the stopping step's own result inside the raised error object (as error.result) instead of discarding it. The top-level result stays { error }. Defaults to false."}},"required":["expr"]},"FlowModule":{"type":"object","description":"A single step in a flow. Can be a script, subflow, loop, or branch","properties":{"id":{"type":"string","description":"Unique identifier for this step. Used to reference results via 'results.step_id'. Must be a valid identifier (alphanumeric, underscore, hyphen)"},"value":{"$ref":"#/components/schemas/FlowModuleValue"},"stop_after_if":{"description":"Early termination condition evaluated after this step completes","$ref":"#/components/schemas/StopAfterIf"},"stop_after_all_iters_if":{"description":"For loops only - early termination condition evaluated after all iterations complete","$ref":"#/components/schemas/StopAfterIf"},"skip_if":{"type":"object","description":"Conditionally skip this step based on previous results or flow inputs","properties":{"expr":{"type":"string","description":"JavaScript expression that returns true to skip. Can use 'flow_input' or 'results.'"}},"required":["expr"]},"sleep":{"description":"Delay before executing this step (in seconds or as expression)","$ref":"#/components/schemas/InputTransform"},"cache_ttl":{"type":"number","description":"Cache duration in seconds for this step's results"},"cache_ignore_s3_path":{"type":"boolean"},"timeout":{"description":"Maximum execution time in seconds (static value or expression)","$ref":"#/components/schemas/InputTransform"},"delete_after_secs":{"type":"integer","description":"If set, delete the step's args, result and logs after this many seconds following job completion"},"summary":{"type":"string","description":"Short description of what this step does"},"mock":{"type":"object","description":"Mock configuration for testing without executing the actual step","properties":{"enabled":{"type":"boolean","description":"If true, return mock value instead of executing"},"return_value":{"description":"Value to return when mocked"}}},"suspend":{"type":"object","description":"Configuration for approval/resume steps that wait for user input","properties":{"required_events":{"type":"integer","description":"Number of approvals required before continuing"},"timeout":{"type":"integer","description":"Timeout in seconds before auto-continuing or canceling"},"resume_form":{"type":"object","description":"Form schema for collecting input when resuming","properties":{"schema":{"type":"object","description":"JSON Schema for the resume form"}}},"user_auth_required":{"type":"boolean","description":"If true, only authenticated users can approve"},"user_groups_required":{"description":"Expression or list of groups that can approve","$ref":"#/components/schemas/InputTransform"},"self_approval_disabled":{"type":"boolean","description":"If true, the user who started the flow cannot approve"},"hide_cancel":{"type":"boolean","description":"If true, hide the cancel button on the approval form"},"continue_on_disapprove_timeout":{"type":"boolean","description":"If true, continue flow on timeout instead of canceling"},"skin":{"type":"string","enum":["detailed","minimal"],"description":"How the approval request is presented, on the approval page and in Slack/Teams approval messages. 'detailed' (used when unset) shows the flow details (arguments, graph, approvers); 'minimal' shows only the request: the step description, form and approve/reject actions"}}},"priority":{"type":"number","description":"Execution priority for this step (higher numbers run first)"},"continue_on_error":{"type":"boolean","description":"If true, flow continues even if this step fails"},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this step's jobs is restricted to (domain:action[:resource], e.g. oidc:write), on top of the flow's own restriction: a step can only narrow it. Applies to everything the step runs (loop and branch bodies, AI agent tools)."},"retry":{"description":"Retry configuration if this step fails","$ref":"#/components/schemas/Retry"},"debouncing":{"description":"Debounce configuration for this step (EE only)","type":"object","properties":{"debounce_delay_s":{"type":"integer","description":"Delay in seconds to debounce this step's executions across flow runs"},"debounce_key":{"type":"string","description":"Expression to group debounced executions. Supports $workspace and $args[name]. Default: $workspace/flow/-"},"debounce_args_to_accumulate":{"type":"array","description":"Array-type arguments to accumulate across debounced executions","items":{"type":"string"}},"max_total_debouncing_time":{"type":"integer","description":"Maximum total time in seconds before forced execution"},"max_total_debounces_amount":{"type":"integer","description":"Maximum number of debounces before forced execution"}}}},"required":["value","id"]},"InputTransform":{"description":"Maps input parameters for a step. Can be a static value or a JavaScript expression that references previous results or flow inputs","oneOf":[{"$ref":"#/components/schemas/StaticTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"StaticTransform":{"type":"object","description":"Static value passed directly to the step. Use for hardcoded values or resource references like '$res:path/to/resource'","properties":{"value":{"description":"The static value. For resources, use format '$res:path/to/resource'"},"type":{"type":"string","enum":["static"]}},"required":["type"]},"JavascriptTransform":{"type":"object","description":"JavaScript expression evaluated at runtime. Can reference previous step results via 'results.step_id' or flow inputs via 'flow_input.property'. Inside for loops, use 'flow_input.iter.value' for the current iteration value (in while loops it equals 'flow_input.iter.index')","properties":{"expr":{"type":"string","description":"JavaScript expression returning the value. Available variables - results (object with all previous step results), flow_input (flow inputs), flow_input.iter (in loops)"},"type":{"type":"string","enum":["javascript"]}},"required":["expr","type"]},"AiTransform":{"type":"object","description":"Value resolved by the AI runtime for this input. The AI engine decides how to satisfy the parameter.","properties":{"type":{"type":"string","enum":["ai"]}},"required":["type"]},"AIProviderKind":{"type":"string","description":"Supported AI provider types","enum":["openai","azure_openai","azure_foundry","anthropic","mistral","deepseek","googleai","groq","openrouter","togetherai","customai","aws_bedrock","typesafe","cloudflare"]},"ProviderConfig":{"type":"object","description":"Complete AI provider configuration with resource reference and model selection","properties":{"kind":{"$ref":"#/components/schemas/AIProviderKind"},"resource":{"type":"string","description":"Resource reference in format '$res:{resource_path}' pointing to provider credentials"},"model":{"type":"string","description":"Model identifier (e.g., 'gpt-4', 'claude-3-opus-20240229', 'gemini-pro')"},"reasoning_effort":{"type":"string","description":"Provider-native reasoning effort token (e.g. 'low', 'high', 'none') for models that support extended thinking. Optional; unset leaves the provider default."}},"required":["kind","resource","model"]},"StaticProviderTransform":{"type":"object","description":"Static provider configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/ProviderConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"ProviderTransform":{"description":"Provider configuration - can be static (ProviderConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticProviderTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticProviderTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"MemoryOff":{"type":"object","description":"No conversation memory/context","properties":{"kind":{"type":"string","enum":["off"]}},"required":["kind"]},"MemoryWindow":{"type":"object","description":"Keeps the most recent messages of the memory named by the run's memory id (or the step's\n`memory_id`). Without a memory id the agent runs without memory.\n","properties":{"kind":{"type":"string","enum":["window"]},"context_length":{"type":"integer","description":"Number of most recent messages to load and store. 0 turns memory off."}},"required":["kind","context_length"]},"MemoryAuto":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written: the run's memory id, else the `memory_id` here.\nThe step's own `memory_id` is not read while this kind is set; switch the kind to `window`\nto use it. Without a `context_length`, or with 0, it is `off` and reads `previous_messages`.\n","properties":{"kind":{"type":"string","enum":["auto"]},"context_length":{"type":"integer","description":"Maximum number of messages to retain in context"},"memory_id":{"type":"string","description":"Identifier for persistent memory across agent invocations"}},"required":["kind"]},"MemoryCompaction":{"type":"object","description":"Keeps the whole memory named by the run's memory id (or the step's `memory_id`), replacing\nits older part with a summary as the conversation approaches the model's context window.\nWithout a memory id the agent runs without memory, and compaction bounds the run's own loop.\n","properties":{"kind":{"type":"string","enum":["compaction"]},"context_window":{"type":"integer","description":"Overrides the context window looked up from the model, in tokens. Only a model\nWindmill does not know needs one; those fall back to 128000.\n"}},"required":["kind"]},"MemoryMessage":{"type":"object","description":"A single message in conversation history","properties":{"role":{"type":"string","enum":["user","assistant","system"]},"content":{"type":"string"}},"required":["role","content"]},"MemoryManual":{"type":"object","deprecated":true,"description":"Deprecated, still read as it was written. Move the step to `off` with `previous_messages` instead.","properties":{"kind":{"type":"string","enum":["manual"]},"messages":{"type":"array","items":{"$ref":"#/components/schemas/MemoryMessage"}}},"required":["kind","messages"]},"MemoryConfig":{"description":"Managed memory, stored by Windmill and replayed with each request. The memory is named by a memory id, see `memory_id`. While it is off, a step can supply its history in `previous_messages`.","oneOf":[{"$ref":"#/components/schemas/MemoryOff"},{"$ref":"#/components/schemas/MemoryWindow"},{"$ref":"#/components/schemas/MemoryCompaction"},{"$ref":"#/components/schemas/MemoryAuto"},{"$ref":"#/components/schemas/MemoryManual"}],"discriminator":{"propertyName":"kind","mapping":{"off":"#/components/schemas/MemoryOff","window":"#/components/schemas/MemoryWindow","compaction":"#/components/schemas/MemoryCompaction","auto":"#/components/schemas/MemoryAuto","manual":"#/components/schemas/MemoryManual"}}},"StaticMemoryTransform":{"type":"object","description":"Static memory configuration passed directly to the AI agent","properties":{"value":{"$ref":"#/components/schemas/MemoryConfig"},"type":{"type":"string","enum":["static"]}},"required":["type","value"]},"MemoryTransform":{"description":"Memory configuration - can be static (MemoryConfig), JavaScript expression, or AI-determined","oneOf":[{"$ref":"#/components/schemas/StaticMemoryTransform"},{"$ref":"#/components/schemas/JavascriptTransform"},{"$ref":"#/components/schemas/AiTransform"}],"discriminator":{"propertyName":"type","mapping":{"static":"#/components/schemas/StaticMemoryTransform","javascript":"#/components/schemas/JavascriptTransform","ai":"#/components/schemas/AiTransform"}}},"FlowModuleValue":{"description":"The actual implementation of a flow step. Can be a script (inline or referenced), subflow, loop, branch, or special module type","oneOf":[{"$ref":"#/components/schemas/RawScript"},{"$ref":"#/components/schemas/PathScript"},{"$ref":"#/components/schemas/PathFlow"},{"$ref":"#/components/schemas/ForloopFlow"},{"$ref":"#/components/schemas/WhileloopFlow"},{"$ref":"#/components/schemas/BranchOne"},{"$ref":"#/components/schemas/BranchAll"},{"$ref":"#/components/schemas/Identity"},{"$ref":"#/components/schemas/AiAgent"},{"$ref":"#/components/schemas/AiDecision"}],"discriminator":{"propertyName":"type","mapping":{"rawscript":"#/components/schemas/RawScript","script":"#/components/schemas/PathScript","flow":"#/components/schemas/PathFlow","forloopflow":"#/components/schemas/ForloopFlow","whileloopflow":"#/components/schemas/WhileloopFlow","branchone":"#/components/schemas/BranchOne","branchall":"#/components/schemas/BranchAll","identity":"#/components/schemas/Identity","aiagent":"#/components/schemas/AiAgent","aidecision":"#/components/schemas/AiDecision"}}},"RawScript":{"type":"object","description":"Inline script with code defined directly in the flow. Use 'bun' as default language if unspecified. The script receives arguments from input_transforms","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"content":{"type":"string","description":"The script source code. Should export a 'main' function"},"language":{"type":"string","description":"Programming language for this script","enum":["deno","bun","bunnative","python3","go","bash","powershell","postgresql","mysql","bigquery","snowflake","mssql","oracledb","graphql","nativets","php","rust","ansible","csharp","nu","java","ruby","rlang","duckdb"]},"path":{"type":"string","description":"Optional path for saving this script"},"lock":{"type":"string","description":"Lock file content for dependencies"},"type":{"type":"string","enum":["rawscript"]},"tag":{"type":"string","description":"Worker group tag for execution routing"},"concurrent_limit":{"type":"number","description":"Maximum concurrent executions of this script"},"concurrency_time_window_s":{"type":"number","description":"Time window for concurrent_limit"},"custom_concurrency_key":{"type":"string","description":"Custom key for grouping concurrent executions"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"},"assets":{"type":"array","description":"External resources this script accesses (S3 objects, resources, etc.)","items":{"type":"object","required":["path","kind"],"properties":{"path":{"type":"string","description":"Path to the asset"},"kind":{"type":"string","description":"Type of asset","enum":["s3object","resource","ducklake","datatable","volume","dbt"]},"access_type":{"type":"string","nullable":true,"description":"Access level for this asset","enum":["r","w","rw"]},"alt_access_type":{"type":"string","nullable":true,"description":"Alternative access level","enum":["r","w","rw"]}}}}},"required":["type","content","language","input_transforms"]},"PathScript":{"type":"object","description":"Reference to an existing script by path. Use this when calling a previously saved script instead of writing inline code","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the script's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the script in the workspace (e.g., 'f/scripts/send_email')"},"hash":{"type":"string","description":"Optional specific version hash of the script to use"},"type":{"type":"string","enum":["script"]},"tag_override":{"type":"string","description":"Override the script's default worker group tag"},"is_trigger":{"type":"boolean","description":"If true, this script is a trigger that can start the flow"}},"required":["type","path","input_transforms"]},"PathFlow":{"type":"object","description":"Reference to an existing flow by path. Use this to call another flow as a subflow","properties":{"input_transforms":{"type":"object","description":"Map of parameter names to their values (static or JavaScript expressions). These become the subflow's input arguments","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}},"path":{"type":"string","description":"Path to the flow in the workspace (e.g., 'f/flows/process_user')"},"type":{"type":"string","enum":["flow"]}},"required":["type","path","input_transforms"]},"ForloopFlow":{"type":"object","description":"Executes nested modules in a loop over an iterator. Inside the loop, use 'flow_input.iter.value' to access the current iteration value, and 'flow_input.iter.index' for the index. Supports parallel execution for better performance on I/O-bound operations","properties":{"modules":{"type":"array","description":"Steps to execute for each iteration. These can reference the iteration value via 'flow_input.iter.value'","items":{"$ref":"#/components/schemas/FlowModule"}},"iterator":{"description":"JavaScript expression that returns an array to iterate over. Can reference 'results.step_id' or 'flow_input'","$ref":"#/components/schemas/InputTransform"},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["forloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (faster for I/O-bound operations). Use with parallelism to control concurrency"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true. Limits resource usage. Can be static number or expression","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","iterator","skip_failures","type"]},"WhileloopFlow":{"type":"object","description":"Executes nested modules repeatedly until stopped. The implicit iterator is the iteration counter, so 'flow_input.iter.value' equals 'flow_input.iter.index' (0, 1, 2, ...) and never carries state. To carry state across iterations, a step reads its own previous-iteration result via 'results.' with a first-iteration fallback - the loop's stop_after_if must then be on that inner step (a plain single-step body with stop_after_if on the loop module does not resolve 'results' across iterations and never terminates); plain counters can instead be derived from 'flow_input.iter.index', which works in every configuration. stop_after_if is evaluated after each iteration - on the loop module 'result' is the last iteration's result","properties":{"modules":{"type":"array","description":"Steps to execute in each iteration","items":{"$ref":"#/components/schemas/FlowModule"}},"skip_failures":{"type":"boolean","description":"If true, iteration failures don't stop the loop. Failed iterations return null"},"type":{"type":"string","enum":["whileloopflow"]},"parallel":{"type":"boolean","description":"If true, iterations run concurrently (use with caution in while loops)"},"parallelism":{"description":"Maximum number of concurrent iterations when parallel=true","$ref":"#/components/schemas/InputTransform"},"squash":{"type":"boolean"}},"required":["modules","skip_failures","type"]},"BranchOne":{"type":"object","description":"Conditional branching where only the first matching branch executes. Branches are evaluated in order, and the first one with a true expression runs. If no branches match, the default branch executes","properties":{"branches":{"type":"array","description":"Array of branches to evaluate in order. The first branch with expr evaluating to true executes","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch condition"},"expr":{"type":"string","description":"JavaScript expression that returns boolean. Can use 'results.step_id' or 'flow_input'. First true expr wins"},"modules":{"type":"array","description":"Steps to execute if this branch's expr is true","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules","expr"]}},"default":{"type":"array","description":"Steps to execute if no branch expressions match","items":{"$ref":"#/components/schemas/FlowModule"}},"type":{"type":"string","enum":["branchone"]}},"required":["branches","default","type"]},"BranchAll":{"type":"object","description":"Parallel branching where all branches execute simultaneously. Unlike BranchOne, all branches run regardless of conditions. Useful for executing independent tasks concurrently","properties":{"branches":{"type":"array","description":"Array of branches that all execute (either in parallel or sequentially)","items":{"type":"object","properties":{"summary":{"type":"string","description":"Short description of this branch's purpose"},"skip_failure":{"type":"boolean","description":"If true, failure in this branch doesn't fail the entire flow"},"modules":{"type":"array","description":"Steps to execute in this branch","items":{"$ref":"#/components/schemas/FlowModule"}}},"required":["modules"]}},"type":{"type":"string","enum":["branchall"]},"parallel":{"type":"boolean","description":"If true, all branches execute concurrently. If false, they execute sequentially"}},"required":["branches","type"]},"AgentTool":{"type":"object","description":"A tool available to an AI agent. Can be a flow module or an external MCP (Model Context Protocol) tool","properties":{"id":{"type":"string","description":"Unique identifier for this tool. Cannot contain spaces - use underscores instead (e.g., 'get_user_data' not 'get user data')"},"summary":{"type":"string","description":"The name the AI agent calls this tool by, not a human label. On a flowmodule tool it must match ^[a-zA-Z0-9_]+$ - letters, numbers and underscores only (e.g. 'search_documentation', not 'Search documentation') - and always be set; on an mcp or websearch tool it is a plain label. Put the human-readable explanation in 'description'."},"description":{"type":"string","description":"Free-text description of the tool given to the AI to decide when and how to call it. Overrides the description auto-derived from the underlying script."},"job_token_scopes":{"type":"array","nullable":true,"items":{"type":"string"},"description":"Scopes the token of this tool's jobs is restricted to, on top of the agent step's own restriction: a tool can only narrow it."},"value":{"$ref":"#/components/schemas/ToolValue"}},"required":["id","value"]},"ToolValue":{"description":"The implementation of a tool. Can be a flow module (script/flow) or an MCP tool reference","oneOf":[{"$ref":"#/components/schemas/FlowModuleTool"},{"$ref":"#/components/schemas/McpToolValue"},{"$ref":"#/components/schemas/WebsearchToolValue"}],"discriminator":{"propertyName":"tool_type","mapping":{"flowmodule":"#/components/schemas/FlowModuleTool","mcp":"#/components/schemas/McpToolValue","websearch":"#/components/schemas/WebsearchToolValue"}}},"FlowModuleTool":{"description":"A tool implemented as a flow module (script, flow, etc.). The AI can call this like any other flow module","allOf":[{"type":"object","properties":{"tool_type":{"type":"string","enum":["flowmodule"]}},"required":["tool_type"]},{"$ref":"#/components/schemas/FlowModuleValue"}]},"WebsearchToolValue":{"type":"object","description":"A tool implemented as a websearch tool. The AI can call this like any other websearch tool","properties":{"tool_type":{"type":"string","enum":["websearch"]}},"required":["tool_type"]},"McpToolValue":{"type":"object","description":"Reference to an external MCP (Model Context Protocol) tool. The AI can call tools from MCP servers","properties":{"tool_type":{"type":"string","enum":["mcp"]},"resource_path":{"type":"string","description":"Path to the MCP resource/server configuration"},"include_tools":{"type":"array","description":"Whitelist of specific tools to include from this MCP server","items":{"type":"string"}},"exclude_tools":{"type":"array","description":"Blacklist of tools to exclude from this MCP server","items":{"type":"string"}}},"required":["tool_type","resource_path"]},"AiAgent":{"type":"object","description":"AI agent step that can use tools to accomplish tasks. The agent receives inputs and can call any of its configured tools to complete the task","properties":{"input_transforms":{"type":"object","description":"Input parameters for the AI agent mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"output_type":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Output format type.\nValid values: 'text' (default) - plain text response, 'image' - image generation\n"},"user_message":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"The user's prompt/message to the AI agent. Supports variable interpolation with\nflow.input syntax. Required unless memory is off and `previous_messages` supplies\nthe prompt; image output always needs it.\n"},"system_prompt":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"System instructions that guide the AI's behavior, persona, and response style. Optional."},"streaming":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Boolean. If true, stream the AI response incrementally.\nStreaming events include: token_delta, reasoning_token_delta, tool_call, tool_call_arguments, tool_execution, tool_result\n"},"memory":{"$ref":"#/components/schemas/MemoryTransform"},"memory_id":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"String. Names the memory this step reads and writes, overriding the memory id the run\nwas started with (the chat conversation, an app chat session or the `memory_id` run\nparameter). Leave unset to use the run's memory id. A fixed value shares one memory\nacross every run; an expression such as `flow_input.customer_id` keeps one memory per\nkey. When it evaluates to an empty value the agent runs without memory. Read only\nwhile `memory` is `window` or `compaction`: it is ignored when memory is off, and an\nolder `auto` or `manual` memory reads neither history input.\n"},"previous_messages":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of MemoryMessage. History supplied by the flow, sent between the system prompt\nand the user message. Read only while `memory` is off or absent: managed memory\nignores it, and an older `auto` or `manual` memory reads neither history input.\n"},"output_schema":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"JSON Schema object defining structured output format. Used when you need the AI to return data in a specific shape.\nSupports standard JSON Schema properties: type, properties, required, items, enum, pattern, minLength, maxLength, minimum, maximum, etc.\nExample: { type: 'object', properties: { name: { type: 'string' }, age: { type: 'integer' } }, required: ['name'] }\n"},"user_attachments":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of file references (images or PDFs) for the AI agent.\nFormat: Array<{ bucket: string, key: string }> - S3 object references\nExample: [{ bucket: 'my-bucket', key: 'documents/report.pdf' }]\n"},"enabled_tools":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Array of strings naming which of the tools configured in `tools` the agent may call\nthis run. Leaving it unset carries every one of them; an empty array carries none.\nA tool is named as the model is shown it. An entry the model is shown nothing of is\nnamed by what identifies it instead: an MCP server by its resource path, carrying\nevery tool it exposes (which of them stays that entry's include_tools/exclude_tools),\nand a websearch entry by the reserved name '__wm_web_search', whatever summary it carries\n(no tool may take that name).\nExample: ['get_user', 'u/admin/github_mcp', '__wm_web_search']\n"},"max_completion_tokens":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Integer. Maximum number of tokens the AI will generate in its response.\nRange: 1 to 4,294,967,295. Typical values: 256-4096 for most use cases.\n"},"temperature":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Float. Controls randomness/creativity of responses.\nRange: 0.0 to 2.0 (provider-dependent)\n- 0.0 = deterministic, focused responses\n- 0.7 = balanced (common default)\n- 1.0+ = more creative/random\n"},"max_iterations":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Number. Limits how many times the agent can loop through reasoning and tool use.\nRange: 1-1000.\n"}}},"tools":{"type":"array","description":"Array of tools the agent can use. The agent decides which tools to call based on the task","items":{"$ref":"#/components/schemas/AgentTool"}},"type":{"type":"string","enum":["aiagent"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the AI agent step runs on the flow's tag (default `flow`)"},"omit_output_from_conversation":{"type":"boolean","default":false,"description":"If true, this AI agent step does not persist its assistant or tool messages to the flow conversation when chat mode is enabled."},"agent":{"type":"string","description":"Path of a reusable `ai_agent` resource (hybrid linking). When set, the agent brain\nconfig (provider/model/system prompt/etc.) and tool set are resolved at runtime from\nthat resource; the module's input_transforms then only carry the flow-local inputs\n(user_message, user_attachments, enabled_tools and the history inputs memory_id and previous_messages).\n"},"tool_inputs":{"type":"object","description":"Host-local wiring for an agent's tool inputs, keyed by tool id then input key. Binds the\nreferenced agent's tools to this flow's context (flow_input/results) without mutating the\nshared resource; overlaid onto the tools' input_transforms at runtime \u2014 including when\n`agent` is unset, since a step forked for editing keeps these overrides until it is saved\nback or unlinked.\n","additionalProperties":{"type":"object","additionalProperties":{"$ref":"#/components/schemas/InputTransform"}}},"parallel":{"type":"boolean","description":"If true, the agent can execute multiple tool calls in parallel"}},"required":["type","input_transforms"]},"AiDecision":{"type":"object","description":"AI decision step: one call to a decision model (TypeSafe's Jev, Cloudflare's\nJev-compatible Clef, or OpenAI's Decisions API) that answers typed questions about a\nstate with calibrated probabilities, instead of generating text.\nPrefer it over an AI agent to classify, route, score or check something: it is fast,\ncheap and its answers are structured. Its result is { output: { : answer },\nmodel, usage }, so a later step reads e.g. results..output.intent.choice. To route on\nthe answers, follow it with a branchone whose branch exprs read\nresults..output..choice. Usable as an AI agent tool, where the calling agent\nsupplies the state.\n","properties":{"input_transforms":{"type":"object","description":"Input parameters for the decision mapped to their values","properties":{"provider":{"$ref":"#/components/schemas/ProviderTransform"},"state":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"What the questions are asked about: a string, an object or an array of strings.\nAn object with descriptive keys holding only what the questions need works best.\nExample: { message: flow_input.message, plan: results.a.plan }\n"},"questions":{"allOf":[{"$ref":"#/components/schemas/InputTransform"}],"description":"Object mapping each question name to { type, instructions, criteria }, where type is:\n- 'choice': criteria maps each option to its description (up to 255); the answer\n has choice, probabilities and confidence\n- 'score': criteria is an ordered array of 2 to 10 level descriptions; the answer\n has score, legend, probabilities and confidence\n- 'noul': yes/no, criteria optionally { true, false } descriptions; the answer has\n noul, the probability of yes (0 to 1)\nExample: { intent: { type: 'choice', instructions: 'What does the customer want?',\ncriteria: { refund: 'Money back', bug: 'Something is broken' } } }\n"}},"required":["provider","state","questions"]},"type":{"type":"string","enum":["aidecision"]},"tag":{"type":"string","description":"Worker group tag for execution routing. If not set, the step runs on the flow's tag (default `flow`)"}},"required":["type","input_transforms"]},"Identity":{"type":"object","description":"Pass-through module that returns its input unchanged. Useful for flow structure or as a placeholder","properties":{"type":{"type":"string","enum":["identity"]},"flow":{"type":"boolean","description":"If true, marks this as a flow identity (special handling)"}},"required":["type"]},"FlowStatus":{"type":"object","properties":{"step":{"type":"integer"},"modules":{"type":"array","items":{"$ref":"#/components/schemas/FlowStatusModule"}},"user_states":{"additionalProperties":true},"preprocessor_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"}]},"failure_module":{"allOf":[{"$ref":"#/components/schemas/FlowStatusModule"},{"type":"object","properties":{"parent_module":{"type":"string"}}}]},"retry":{"type":"object","properties":{"fail_count":{"type":"integer"},"failed_jobs":{"type":"array","items":{"type":"string","format":"uuid"}}}}},"required":["step","modules","failure_module"]},"FlowStatusModule":{"type":"object","properties":{"type":{"type":"string","enum":["WaitingForPriorSteps","WaitingForEvents","WaitingForExecutor","InProgress","Success","Failure"]},"id":{"type":"string"},"job":{"type":"string","format":"uuid"},"count":{"type":"integer"},"progress":{"type":"integer"},"iterator":{"type":"object","properties":{"index":{"type":"integer"},"itered":{"type":"array","items":{}},"itered_len":{"type":"integer"},"args":{}}},"flow_jobs":{"type":"array","items":{"type":"string"}},"flow_jobs_success":{"type":"array","items":{"type":"boolean"}},"flow_jobs_duration":{"type":"object","properties":{"started_at":{"type":"array","items":{"type":"string"}},"duration_ms":{"type":"array","items":{"type":"integer"}}}},"branch_chosen":{"type":"object","properties":{"type":{"type":"string","enum":["branch","default"]},"branch":{"type":"integer"}},"required":["type"]},"branchall":{"type":"object","properties":{"branch":{"type":"integer"},"len":{"type":"integer"}},"required":["branch","len"]},"approvers":{"type":"array","items":{"type":"object","properties":{"resume_id":{"type":"integer"},"approver":{"type":"string"}},"required":["resume_id","approver"]}},"failed_retries":{"type":"array","items":{"type":"string","format":"uuid"}},"skipped":{"type":"boolean"},"agent_actions":{"type":"array","items":{"type":"object","oneOf":[{"type":"object","properties":{"job_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"type":{"type":"string","enum":["tool_call"]},"module_id":{"type":"string"}},"required":["job_id","function_name","type","module_id"]},{"type":"object","properties":{"call_id":{"type":"string","format":"uuid"},"function_name":{"type":"string"},"resource_path":{"type":"string"},"type":{"type":"string","enum":["mcp_tool_call"]},"arguments":{"type":"object"}},"required":["call_id","function_name","resource_path","type"]},{"type":"object","properties":{"type":{"type":"string","enum":["web_search"]}},"required":["type"]},{"type":"object","properties":{"type":{"type":"string","enum":["message"]}},"required":["content","type"]}]}},"agent_actions_success":{"type":"array","items":{"type":"boolean"}}},"required":["type"]}} diff --git a/system_prompts/auto-generated/skills/write-pipeline/SKILL.md b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md index 064dc600ac..4c339e6222 100644 --- a/system_prompts/auto-generated/skills/write-pipeline/SKILL.md +++ b/system_prompts/auto-generated/skills/write-pipeline/SKILL.md @@ -12,10 +12,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- `bun` / `python3` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with `wmill.ducklake()`, or into a data table with `wmill.datatable()`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +```python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +``` + +```ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)`.fetch() +} +``` + +- Use `INSERT INTO
SELECT ...` instead of `CREATE OR REPLACE` to append (an incremental pull), with a one-time `CREATE TABLE IF NOT EXISTS` first. +- Assign the client to a variable (`lake = wmill.ducklake()`) and keep the SQL a **string literal** (Python) or a template without `sql.raw` (TS): that is what lets the graph record the node's write to `ducklake://main/
`. A chained `wmill.ducklake().query(...)` or SQL built in a variable records no output, so the downstream edge never forms. +- `wmill.ducklake("")` targets a non-default catalog; `wmill.datatable("")` writes a Postgres data table the same way (Python `db.query(sql, *args)` with `$1` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: `-- on ducklake://main/stripe_charges`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -59,7 +97,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the table itself with `wmill.ducklake()` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use `duckdb` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. diff --git a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md index 4b340c0db1..055b1613b2 100644 --- a/system_prompts/auto-generated/skills/write-script-bun/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bun/SKILL.md @@ -320,21 +320,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -390,6 +392,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -649,6 +659,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md index 03d40ab044..beaf7817bb 100644 --- a/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-bunnative/SKILL.md @@ -332,21 +332,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -402,6 +404,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -661,6 +671,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md index b15655a6da..afefb7e987 100644 --- a/system_prompts/auto-generated/skills/write-script-deno/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-deno/SKILL.md @@ -322,21 +322,23 @@ async cancelJob(jobId: string, reason: string | undefined = undefined): Promise< * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByPathAsync(path: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ -async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null = null, tag: string | null = null): Promise +async runScriptByHashAsync(hash_: string, args: Record | null = null, scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null): Promise /** * Run a flow asynchronously by its path @@ -392,6 +394,14 @@ async setProgress(percent: number, jobId?: any): Promise */ async getProgress(jobId?: any): Promise +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +async workerIsDraining(jobId?: string): Promise + /** * Set a flow user state * @param key key of the state @@ -651,6 +661,9 @@ async sleep(seconds: number): Promise * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ async step(name: string, fn: () => T | Promise,): Promise>> diff --git a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md index b9c2793f47..ee50b6d62d 100644 --- a/system_prompts/auto-generated/skills/write-script-python3/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-python3/SKILL.md @@ -268,10 +268,14 @@ def post(endpoint, raise_for_status = True, **kwargs) -> httpx.Response def create_token(duration = dt.timedelta(days=1)) -> str # Create a script job by path and return its job id. -def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# +# ``retry`` re-runs the job when it fails, with the policy a schedule takes, +# e.g. ``{"constant": {"attempts": 3, "seconds": 10}}``. The returned id is +# the first attempt's; each retry is a job of its own, recorded as a child of it. +def run_script_by_path_async(path: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str -# Create a script job by hash and return its job id. -def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None) -> str +# Create a script job by hash and return its job id. See ``run_script_by_path_async`` for ``retry``. +def run_script_by_hash_async(hash_: str, args: dict = None, scheduled_in_secs: int = None, tag: str = None, retry: dict = None) -> str # Create a flow job and return its job id. def run_flow_async(path: str, args: dict = None, scheduled_in_secs: int = None, do_not_track_in_parent: bool = True, tag: str = None) -> str @@ -444,6 +448,18 @@ def set_progress(value: int, job_id: Optional[str] = None) # Progress value (0-100) or None if not set def get_progress(job_id: Optional[str] = None) -> Any +# Whether the worker running the job has received its shutdown signal. +# +# A draining worker waits for its current job to finish before exiting. A long-running +# script can poll this and return early so the worker can be replaced. +# +# Args: +# job_id: Job ID (defaults to current WM_JOB_ID) +# +# Returns: +# True if the worker running the job is draining +def worker_is_draining(job_id: Optional[str] = None) -> bool + # Set the user state of a flow at a given key def set_flow_user_state(key: str, value: Any) -> None @@ -861,7 +877,9 @@ def parse_sql_client_name(name: str) -> tuple[str, Optional[str]] # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -947,12 +965,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md index 8c438e7038..cd79a1f5d4 100644 --- a/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md +++ b/system_prompts/auto-generated/skills/write-script-snowflake/SKILL.md @@ -84,6 +84,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Array parameters + +Declare a list with a `[]` suffix on the element type (`(varchar[])`, +`(int[])`), or `(array)` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +`PARSE_JSON(?)`, and cast to `ARRAY` where a function expects one. + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +``` + +Or join on the elements with `FLATTEN`: + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +``` + ## Receiving an S3Object as a script parameter Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for diff --git a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md index 13d305c74e..09789b388a 100644 --- a/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md +++ b/system_prompts/auto-generated/skills/write-workflow-as-code/SKILL.md @@ -143,6 +143,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with `@task()` or `@task`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or `step()` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. `sleep()` there waits in the job, and `taskScript()` / `taskFlow()` / `waitForApproval()` (`task_script()` / `task_flow()` / `wait_for_approval()`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: ```typescript @@ -308,8 +310,9 @@ export async function getResumeUrls(approver?: string, flowLevel?: boolean): Pro * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -353,6 +356,9 @@ export function workflow(fn: (...args: any[]) => Promise) * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step(name: string, fn: () => T | Promise,): Promise>> @@ -451,7 +457,9 @@ def get_resume_urls(approver: str = None, flow_level: bool = None) -> dict # # - **v2 (inside @workflow)**: dispatches as a checkpoint step. # - **v1 (WM_JOB_ID set, no @workflow)**: dispatches via HTTP API. -# - **Standalone**: executes the function body directly. +# - **Standalone, or called from inside another v2 task**: executes the +# function body directly. A task runs as a single job, so the tasks it +# calls run in that same job, without their own options. # # A task runs as its own job, so its result is always encoded as JSON and # decoded back before the caller sees it: a ``datetime`` comes back as a @@ -537,12 +545,15 @@ def workflow(func) # ``fn``'s result is encoded as JSON and decoded back before it is returned, # so the round that runs the body sees the same types every replay sees: # a ``datetime`` comes back as a string, a tuple as a list. +# +# Called from inside a task body, ``fn`` runs in place and nothing is +# checkpointed: a task is a single job and is never replayed. async def step(name: str, fn) # Server-side sleep — suspend the workflow for the given duration without holding a worker. # # Inside a @workflow, the parent job suspends and auto-resumes after ``seconds``. -# Outside a workflow, falls back to ``asyncio.sleep``. +# Outside a workflow, or inside a task body, falls back to ``asyncio.sleep``. async def sleep(seconds: int) # Suspend the workflow and wait for an external approval. diff --git a/system_prompts/base/datatable.md b/system_prompts/base/datatable.md new file mode 100644 index 0000000000..c90fb5e349 --- /dev/null +++ b/system_prompts/base/datatable.md @@ -0,0 +1,43 @@ +# Data tables: schema changes, migrations and permissions + +A data table is a PostgreSQL database managed by Windmill and shared by the whole workspace: scripts, flows and apps reach it by name, `main` being the default. It is configured in the workspace settings (Data Tables), never created by SQL. + +## Changing the schema + +A data table with migrations enabled keeps its schema as an ordered list of versioned SQL migrations, each an `up` and an optional `down`. That list is what git sync and a workspace merge carry, and what rebuilds the schema in another workspace. So first find out whether migrations are enabled on the data table: + +- **Enabled**: make every schema change (CREATE, ALTER, DROP, anything but row data) as a new migration. A change made with plain SQL exists in that one database only, and the history no longer describes it. +- **Not enabled**: schema changes are plain SQL. Enabling migrations is a workspace admin's decision, made in the Data Tables settings, not a step to take on the way to something else. + +Writing a migration: + +- Wrap the statements in `BEGIN; ... END;` so it applies atomically. Leave the frame out only for a statement that cannot run in a transaction, such as `CREATE INDEX CONCURRENTLY`: such a migration can fail half applied. +- Give it a `down` that undoes the `up` whenever the change is reversible. Without one it cannot be rolled back. +- Migrations apply oldest first. One that is applied is not edited: correct it with a new migration. +- Row data is not schema. Insert, update and delete rows with plain SQL, migrations or not. + + +`wmill datatable migrate new ` scaffolds `migrations/datatable//_.up.sql` and `.down.sql`, `wmill datatable migrate up` pushes the local files and applies what is pending, `wmill datatable migrate down` rolls back the most recent one and `wmill datatable migrate status` lists them. All take `-d ` and default to `main`. + + +## A migration is not a draft + +A migration is saved to the workspace and changes the database as soon as it is applied, for everything that reads those tables. To try a schema out without touching tables already in use: + +- put the new tables in a schema of their own, created by the same migration (`CREATE SCHEMA IF NOT EXISTS`), and point what you are building at it. A raw app names its schema in its `data` config, which is why a new app gets a new schema by default; +- or work in a workspace fork whose data table was cloned when the fork was created ("Clone schema only" or "Clone schema and data"): migrations applied there change that copy alone, and reach the parent workspace when the fork is merged. A fork that kept the original data table points at the parent's database, so a migration applied in it changes the parent's tables at once. + +Altering or dropping a table other scripts or apps use is neither of those: say what depends on it before doing it. + +## Roles and permissions + +Data table roles are a self-hosted Enterprise Edition feature, for data tables on managed Postgres (the instance database or the external managed instance). Without them, and always on a data table backed by a Postgres resource, everyone who can use the data table reaches all of it: there is no per-user, per-schema or per-row access to promise. + +A data table under roles is reached as a Postgres role, and Postgres enforces what that role was granted: + +- roles are assigned, per data table, to users, groups or folders. `admin` is the connection the data table had before roles: it owns the existing objects and ignores grants; +- a connection either names a role (`wmill.datatable('main', { role: 'analyst' })`, `role='analyst'` in Python, `datatable://main?role=analyst`) and holds that role alone, or names none and is resolved from the data table's default role; +- `permission denied` means the role in use lacks a grant. Say which role and which object: the grant is a data table admin's to give, in the roles drawer of the Database Manager. Do not switch to `admin` to get past it; +- a migration runs as the role its SQL declares on its first line, `-- role `, the same one in its `down`. One that declares none runs as `admin`, which only those who may use `admin` can do. + +Read the Permissions section of https://www.windmill.dev/docs/core_concepts/persistent_storage/data_tables before designing who reaches what: it covers tenants, grants, the default role and the roles a connection that names none ends up holding, rows that differ per viewer through row-level security, and how forks and raw apps behave under roles. diff --git a/system_prompts/base/flow-base.md b/system_prompts/base/flow-base.md index ddb10fd0fa..ff10f38062 100644 --- a/system_prompts/base/flow-base.md +++ b/system_prompts/base/flow-base.md @@ -289,6 +289,10 @@ can call one, not an agent used as a tool. - `results.step_id.property` - Access specific property from a previous step output only when that step result is in scope - `flow_input.iter.value` - Current iteration value inside a `forloopflow`; in a `whileloopflow` it is just the iteration index (a plain number, same as `flow_input.iter.index`) - `flow_input.iter.index` - Current loop index when inside a loop (`forloopflow` or `whileloopflow`) +- After a suspend step (approval), the next step's javascript expressions also get: + - `resume` - the last resume payload (the resume form values); with `continue_on_disapprove_timeout`, a disapproval or timeout yields `{ error: { name: "SuspendedDisapproved" | "SuspendedTimedOut", message } }` instead + - `resumes` - every resume payload, in arrival order + - `approvers` - a plain array of strings, one per resume: for a resume through the `resume`/`cancel` URL from `getResumeUrls(approver)`, the `approver` passed there if any; otherwise (including the `approvalPage` link) the resuming user's username, else `"anonymous"`, e.g. `approvers[0]`. It is NOT the `{ resume_id, approver }` objects stored in the flow status, so `approvers[0].approver` is always undefined ## Loop Structure Rules diff --git a/system_prompts/base/pipeline-base.md b/system_prompts/base/pipeline-base.md index 01e9797e32..59924faf41 100644 --- a/system_prompts/base/pipeline-base.md +++ b/system_prompts/base/pipeline-base.md @@ -7,10 +7,48 @@ A **data pipeline** is NOT a flow. A flow is one runnable that orchestrates step A pipeline node that produces a table should almost always be a **`duckdb`** node that materializes its output into a **DuckLake** table with `-- materialize ducklake:///
` (in a DuckDB node the annotation uses SQL `--` comment syntax; write the body as a bare `SELECT` and let the runtime do the write). DuckLake is the default lakehouse store for pipelines and is the shape the pipeline editor is built around, so prefer it unless the work specifically calls for something else: - `postgresql` / data tables — only for row-level, OLTP-style mutations against an existing Postgres data table (frequent single-row upserts/updates, transactional reads that an app queries live). -- `bun` / `python3` — only for non-tabular work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node still produces tabular data for downstream steps, land it in DuckLake (write it with the wmill SDK / ducklake helpers) rather than inventing a parallel store. +- `bun` / `python3` — only for work that doesn't map to SQL: calling an external API, wrangling files, arbitrary glue. When such a node produces tabular data for downstream steps, write it **straight into DuckLake** (or a data table) from the same script — see "Ingesting from Python / TypeScript" below — rather than inventing a parallel store. Do not spread a pipeline across postgres, S3, and DuckLake when one DuckLake lake would do; a consistent DuckLake lakehouse is the goal. +## Ingesting from Python / TypeScript + +A node that fetches data in code (an API, a SaaS export, a scraper) writes the rows **directly** into their DuckLake table with `wmill.ducklake()`, or into a data table with `wmill.datatable()`. DuckLake already stores every table as parquet files in the workspace object storage, so **never stage the raw payload first** — no intermediate JSON / CSV / parquet file in S3 and no second "load the raw file" node. One ingestion node, one table. + +Pass the rows as a query argument (a list of dicts / array of objects is sent as JSON) and let SQL unnest them; declare the column types so the table gets a real schema: + +```python +# pipeline +# on schedule +import wmill + +def main(): + rows = fetch_charges() # list[dict] + lake = wmill.ducklake() + lake.query(""" + CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json($rows, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true) + """, rows=rows).fetch() +``` + +```ts +// pipeline +// on schedule +import * as wmill from "windmill-client" + +export async function main() { + const rows = await fetchCharges() + const lake = wmill.ducklake() + await lake`CREATE OR REPLACE TABLE stripe_charges AS + SELECT unnest(from_json(${rows}, '[{"id":"VARCHAR","amount":"BIGINT","created":"BIGINT"}]'), recursive := true)`.fetch() +} +``` + +- Use `INSERT INTO
SELECT ...` instead of `CREATE OR REPLACE` to append (an incremental pull), with a one-time `CREATE TABLE IF NOT EXISTS` first. +- Assign the client to a variable (`lake = wmill.ducklake()`) and keep the SQL a **string literal** (Python) or a template without `sql.raw` (TS): that is what lets the graph record the node's write to `ducklake://main/
`. A chained `wmill.ducklake().query(...)` or SQL built in a variable records no output, so the downstream edge never forms. +- `wmill.ducklake("")` targets a non-default catalog; `wmill.datatable("")` writes a Postgres data table the same way (Python `db.query(sql, *args)` with `$1` placeholders, TS the same template form) — use it only when the data belongs in Postgres (see above). +- The downstream node triggers on the table itself: `-- on ducklake://main/stripe_charges`. + ## Storage prerequisites A DuckLake pipeline only runs once the workspace has **object storage** (S3 / Azure Blob / GCS) **and a DuckLake catalog** configured — DuckLake tables and `s3://` assets can't be materialized or read without it. Check which DuckLake catalogs the workspace has before you build. @@ -56,7 +94,7 @@ The key must be a **string literal** — the graph parser is static and cannot f ## Materialize (the managed output) -> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the output via the SDK (`wmill.writeS3File(...)`, ducklake helpers, …) and let it be inferred. Use `duckdb` when a node should materialize a DuckLake table. +> **A MANAGED `// materialize` is DuckDB-only**, and its target must be a DuckLake table (`ducklake:///
`). Deploy **rejects** a `ducklake://` `// materialize` on any other language (`python3`, `bun`, `postgresql`). For a non-DuckDB node writing the lake, do **not** use `// materialize` — write the table itself with `wmill.ducklake()` (see "Ingesting from Python / TypeScript") and let the output be inferred. Use `duckdb` when a node should materialize a DuckLake table. > > The one target ANY language may declare (except a dbt script, whose writes come from its manifest) is a **warehouse relation**: `// materialize manual dbt:////`, where `` is a warehouse the workspace configures under Settings → dbt. `manual` is the only mode it has — nothing generates warehouse DDL, so the node issues its own write (a postgresql `CREATE TABLE` / `INSERT`, an SDK load, …) and the annotation records the outcome. Use it on an ingestion node whose output a dbt project reads as a `source`: the declared relation and the dbt model land on ONE graph node, and a downstream `// on dbt:////` fires when the ingestion node completes. diff --git a/system_prompts/base/raw-app-cli.md b/system_prompts/base/raw-app-cli.md index 59ef0acb58..e22de102f6 100644 --- a/system_prompts/base/raw-app-cli.md +++ b/system_prompts/base/raw-app-cli.md @@ -12,7 +12,7 @@ You need three values to run the command: 1. **summary** — a short description of the app 2. **path** — the windmill path, e.g. `f/folder/my_app` or `u/username/my_app` -3. **framework** — one of `react19` (recommended), `react18`, `svelte5`, `vue` +3. **framework** — one of `react19` (recommended), `react18`, `svelte5` **summary and path**: if the user's request did not supply them, ask. Do not guess values and do not invent paths. @@ -84,7 +84,7 @@ my_app__raw_app/ ├── DATATABLES.md # Database schemas (run 'wmill app generate-agents' to refresh) ├── raw_app.yaml # App configuration (summary, path, data settings) ├── index.tsx # Frontend entry point -├── App.tsx # Main React/Svelte/Vue component +├── App.tsx # Main React/Svelte component ├── index.css # Styles ├── package.json # Frontend dependencies ├── wmill.ts # Auto-generated backend type definitions (DO NOT EDIT) @@ -183,7 +183,7 @@ data: - `/
` — Specific table in public schema - `/:
` — Table in specific schema -**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role. +**Roles:** when a datatable is under roles, its queries run as a role, which only reaches what it was granted. `roles` records the role the app uses each datatable through; the app's code must pass the same role: `wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python. A datatable without an entry is used as its default role, if it has one, plus the roles of whoever is using the app unless that default is `admin`. ## SQL Migrations (sql_to_apply/) diff --git a/system_prompts/base/raw-app.md b/system_prompts/base/raw-app.md index 03bde6ec71..0979000b2a 100644 --- a/system_prompts/base/raw-app.md +++ b/system_prompts/base/raw-app.md @@ -1,6 +1,6 @@ # Windmill Raw Apps -Raw apps let you build custom frontends with React, Svelte, or Vue that connect to Windmill backend runnables and datatables. +Raw apps let you build custom frontends with React or Svelte that connect to Windmill backend runnables and datatables. ## App shape @@ -16,7 +16,7 @@ A new app gets the setup Windmill's own new-app dialog gives one: a framework, t ### Framework -Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18, Svelte 5 and Vue are for when they are asked for. +Use the framework the user named. When they named none, follow what there is to follow: the app being edited, or the apps the request points at. With nothing to follow, use **React 19**, the recommended default, **without asking**. React 18 and Svelte 5 are for when they are asked for. These three are the only frameworks the raw app bundler builds. When the user asks to build or convert an app in another framework, write no code in it: say it is not available and offer these three instead. ### Data setup @@ -54,7 +54,7 @@ An app the user just started from the new-app dialog is already set up: its file ### Entrypoint -The entrypoint is `index.tsx` for React and `index.ts` for Svelte and Vue. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty `
` and auto-renders nothing, so the entrypoint must mount a top-level `App` itself. Keep the UI in `App.tsx` / `App.svelte` / `App.vue` and keep the entrypoint as the mount shim. +The entrypoint is `index.tsx` for React and `index.ts` for Svelte. It is both the bundling entrypoint (the bundler is esbuild) and the **mount** entrypoint: the preview executes the bundle against an empty `
` and auto-renders nothing, so the entrypoint must mount a top-level `App` itself. Keep the UI in `App.tsx` / `App.svelte` and keep the entrypoint as the mount shim. React (`index.tsx`): @@ -66,7 +66,7 @@ import App from './App' createRoot(document.getElementById('root')!).render() ``` -Svelte (`index.ts`): `mount(App, { target: document.getElementById('root')! })`. Vue (`index.ts`): `createApp(App).mount('#root')`. +Svelte (`index.ts`): `mount(App, { target: document.getElementById('root')! })`. **Never replace the entrypoint with a bare component** (`export default function App() { ... }` and no mount call). A component that is defined but never mounted renders a blank screen with **no error thrown** — it never executes, so nothing reaches the console or the error overlay. If an app renders blank, check that the entrypoint still mounts `App` into `#root`. @@ -139,16 +139,16 @@ Apply it to customer data, internal notes and anything else a viewer of the demo ### Chat UIs over a flow in chat mode -A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `windmill-chat` to `package.json` and use it directly, it detects the app's Windmill and credential. +A flow deployed with chat mode on is a chat backend (streaming answer, tool calls, memory, conversation history). Do not drive it through a runnable: add `@windmill-labs/chat-sdk` to `package.json` and use it directly, it detects the app's Windmill and credential. ```tsx -import { useWindmillChat } from 'windmill-chat/react' +import { useWindmillChat } from '@windmill-labs/chat-sdk/react' const chat = useWindmillChat({ flowPath: 'f/support/assistant' }) // chat.messages ({ role, content, pending, success, tool? }), chat.status, chat.sendMessage(text), chat.stop() ``` -`windmill-chat/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `windmill-chat/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. +`@windmill-labs/chat-sdk/ai-sdk` gives a `ChatTransport` for the Vercel AI SDK's `useChat`, `@windmill-labs/chat-sdk/assistant-ui` a runtime for assistant-ui. The flow must be deployed, not a draft. A sandboxed app needs `jobs:run` in its frontend SDK scopes, plus `flow_conversations:write` for the conversation sidebar; without them the chat keeps history in the browser. ## Backend runnables @@ -239,6 +239,20 @@ Either one lets those people run the app's backend runnables, so never open an a When a deploy widens who may open the app, tell the user in plain words. +## Sandbox isolation and frontend API access + +By default a deployed app's frontend runs with the viewer's own Windmill session. A sandboxed app is isolated from it: the frontend reaches Windmill only through its backend runnables, plus whatever frontend API scopes the app declares, which each viewer approves before the app runs. The scopes are `jobs:run`, `jobs:read`, `users:read`, `resources:read`, `variables:read`, `flow_conversations:read` and `flow_conversations:write`; grant only the ones the frontend calls. + +In `raw_app.yaml`: + +```yaml +sandbox: true +frontend_api: [jobs:run, users:read] +``` + +A file with a `sandbox` key states both settings: `sandbox: false` turns isolation off, and a missing `frontend_api` means no scopes. A file without the key leaves the deployed app's isolation and scopes as they are, so removing `sandbox: true` does not unsandbox it. `frontend_api` needs `sandbox: true`; the push fails otherwise. + + ## Data Tables Data tables are PostgreSQL databases managed by Windmill. Backend runnables query them via the `wmill` client; the frontend never queries them directly. **When the app needs to store or persist data** (user data, settings, application state, records, logs), use a data table. @@ -250,6 +264,7 @@ Data tables are PostgreSQL databases managed by Windmill. Backend runnables quer 3. **No DDL inside runnables**: runnables only read and write rows (SELECT, INSERT, UPDATE, DELETE) on existing tables. Never CREATE, ALTER or DROP a table from a runnable. 4. **Qualify table names**: an unqualified name means the `public` schema, so write every other table as `schema.table`, in table creation and queries alike. The app's `data` config sets the default datatable and schema its tables go in; use them consistently across runnables. 5. **Pass the role**: when the app's `data.roles` gives a data table a role, every `wmill.datatable` call on it passes that role (`wmill.datatable('main', { role: 'analyst' })` in TypeScript, `wmill.datatable('main', role='analyst')` in Python). The role only reaches what it was granted, so a query outside it fails with `permission denied`. +6. **Per-viewer rows**: a data table with no entry in `data.roles` is called with no role. Unless its default role is `admin`, the connection then holds the default role, if it has one, plus every role the person using the app is a tenant of, and is refused when that is none. Row-level security policies written as `pg_has_role(current_user, '', 'member')` return that viewer's rows with nothing added to the runnable. A role pins the connection to that one role for every viewer, so where rows should differ per viewer give the data table no `data.roles` entry. `wmill datatable list` lists the workspace's data tables. Create or change tables with a migration in `sql_to_apply/` (see "SQL Migrations" above), then add them to `data.tables` in `raw_app.yaml`. @@ -305,4 +320,4 @@ def main(user_id: str): 7. **Mark sensitive UI with `data-wm-no-record`** — it is what keeps that data out of a recorded demo; passwords are handled for you. 8. **Reach for `backendAsync` + `waitJob`** for long work — never a hand-written job-polling runnable. 9. **Deploy what a path runnable points at** — a path runnable aimed at a draft fails at runtime; tell the user what needs deploying. -10. **Use `windmill-chat` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. +10. **Use `@windmill-labs/chat-sdk` for a chat over a chat-mode flow** — never a runnable that runs the flow and polls its stream. diff --git a/system_prompts/base/workflow-as-code.md b/system_prompts/base/workflow-as-code.md index 0e529f6803..efc29df360 100644 --- a/system_prompts/base/workflow-as-code.md +++ b/system_prompts/base/workflow-as-code.md @@ -95,6 +95,8 @@ async def enrich(customer_id: str): In TypeScript, prefer assigning each task to a named top-level const. In Python, prefer top-level async functions decorated with `@task()` or `@task`. +Only the workflow body dispatches steps. A task runs as a single job, so a task or `step()` called from inside a task body runs in place in that job, as a plain function call: no child job, no checkpoint, and its task options (retry, timeout, tag) do not apply. `sleep()` there waits in the job, and `taskScript()` / `taskFlow()` / `waitForApproval()` (`task_script()` / `task_flow()` / `wait_for_approval()`) throw. To get separate jobs, call the tasks one after another from the workflow body and pass results between them. + For existing scripts: ```typescript diff --git a/system_prompts/generate.py b/system_prompts/generate.py index 918deef0fb..055f03d543 100644 --- a/system_prompts/generate.py +++ b/system_prompts/generate.py @@ -1696,6 +1696,12 @@ TOPICS: dict[str, dict] = { 'chat_helper': 'getPipelinePrompt', 'parts': ['pipeline-base.md'], }, + 'datatable': { + 'skill': 'datatables', + 'description': "MUST use when changing a data table's schema, writing or running its migrations, or working with its roles and permissions.", + 'chat_helper': 'getDatatablePrompt', + 'parts': ['datatable.md'], + }, 'cli_commands': { 'skill': 'cli-commands', 'description': 'MUST use when using the CLI, including debugging job failures and inspecting run history via `wmill job`.', @@ -1715,6 +1721,7 @@ CHAT_CONSTANTS = { 'resources.md': 'RESOURCES_BASE', 'raw-app.md': 'RAW_APP_BASE', 'pipeline-base.md': 'PIPELINE_BASE', + 'datatable.md': 'DATATABLE_BASE', 'workflow-as-code.md': 'WORKFLOW_AS_CODE_BASE', 'flow-chat-special-modules.md': 'FLOW_CHAT_SPECIAL_MODULES', } @@ -2650,6 +2657,11 @@ export function getPipelinePrompt(): string { __RETURN_pipeline__ } +// Helper for data table schema changes, migrations and permissions (chat consumers) +export function getDatatablePrompt(): string { + __RETURN_datatable__ +} + // Helper to get the datatable SQL SDK reference (wmill.datatable()). // Pass a language to get only that SDK; omit it to get both. export function getDatatableSdkReference(language?: string): string { @@ -2711,6 +2723,7 @@ export declare function getFlowPrompt(): string; export declare function getResourcePrompt(): string; export declare function getRawAppPrompt(language?: string): string; export declare function getPipelinePrompt(): string; +export declare function getDatatablePrompt(): string; export declare function getDatatableSdkReference(language?: string): string; export declare function getWorkflowAsCodePrompt(language?: string): string; """ diff --git a/system_prompts/languages/snowflake.md b/system_prompts/languages/snowflake.md index 8ded76eb3c..b7798e3461 100644 --- a/system_prompts/languages/snowflake.md +++ b/system_prompts/languages/snowflake.md @@ -10,6 +10,27 @@ Name the parameters by adding comments before the statement: SELECT * FROM users WHERE name = ? AND age > ?; ``` +## Array parameters + +Declare a list with a `[]` suffix on the element type (`(varchar[])`, +`(int[])`), or `(array)` for an untyped one. Snowflake cannot bind an array +directly, so Windmill binds the list as JSON text: wrap the placeholder with +`PARSE_JSON(?)`, and cast to `ARRAY` where a function expects one. + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +WHERE ARRAY_CONTAINS(o.owner_group::VARIANT, PARSE_JSON(?)::ARRAY); +``` + +Or join on the elements with `FLATTEN`: + +```sql +-- ? groups (varchar[]) +SELECT o.* FROM orders o +JOIN TABLE(FLATTEN(input => PARSE_JSON(?))) g ON g.value::STRING = o.owner_group; +``` + ## Receiving an S3Object as a script parameter Declare the arg with type `(s3object)`. Windmill renders an S3 file picker for diff --git a/typescript-client/build.jsr.sh b/typescript-client/build.jsr.sh index e1ad956dc4..41a8eac37e 100755 --- a/typescript-client/build.jsr.sh +++ b/typescript-client/build.jsr.sh @@ -22,6 +22,6 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, workerIsDraining, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, type RunScriptAsyncOptions, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI } from "./client";' >> "${script_dirpath}/src/index.ts" diff --git a/typescript-client/build.sh b/typescript-client/build.sh index 8b51f75dfc..f927acd26e 100755 --- a/typescript-client/build.sh +++ b/typescript-client/build.sh @@ -46,7 +46,7 @@ cp "${script_dirpath}/sqlUtils.ts" "${script_dirpath}/src/" echo "" >> "${script_dirpath}/src/index.ts" echo 'export type { DenoS3LightClientSettings } from "./s3Types";' >> "${script_dirpath}/src/index.ts" echo "" >> "${script_dirpath}/src/index.ts" -echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" +echo 'export { type Base64, setClient, getWorkspace, workerHasInternalServer, SHARED_FOLDER, getStatePath, getInternalState, setInternalState, getResumeEndpoints, getResult, getResultMaybe, resolveDefaultResource, databaseUrlFromResource, base64ToUint8Array, uint8ArrayToBase64, getVariable, setVariable, getResource, setResource, getResumeUrls, setState, setProgress, getProgress, workerIsDraining, getState, getIdToken, denoS3LightClientSettings, cancelJob, loadS3FileStream, loadS3File, writeS3File, deleteS3File, signS3Objects, signS3Object, getPresignedS3PublicUrls, getPresignedS3PublicUrl, task, taskScript, taskFlow, workflow, step, sleep, parallel, waitForApproval, getApprovalUrls, type TaskOptions, type TaskRetry, type Jsonified, type JsonifiedFn, WorkflowCtx, _workflowCtx, setWorkflowCtx, StepSuspend, runScript, runScriptAsync, runScriptByPath, runScriptByHash, runScriptByPathAsync, runScriptByHashAsync, type RunScriptAsyncOptions, runFlow, runFlowAsync, waitJob, getRootJobId, setFlowUserState, getFlowUserState, usernameToEmail, requestInteractiveSlackApproval, type Sql, requestInteractiveTeamsApproval, appendToResultStream, streamResult, datatable, ducklake, upsertPartition, appendPartition, type DucklakeMaterializeOptions, type SqlStatement, type DatatableSqlTemplateFunction, type SqlTemplateFunction, type S3Object, type S3ObjectRecord, type S3ObjectURI, commitKafkaOffsets } from "./client";' >> "${script_dirpath}/src/index.ts" # Build default export by combining client utilities + services # This preserves backward compatibility for `import wmill from "windmill-client"` @@ -63,6 +63,7 @@ import { setState, setProgress, getProgress, + workerIsDraining, getState, getIdToken, denoS3LightClientSettings, @@ -153,6 +154,7 @@ const wmill = { setState, setProgress, getProgress, + workerIsDraining, getState, getIdToken, denoS3LightClientSettings, diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 7e84320346..da957239cf 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -11,6 +11,7 @@ import { UserService, KafkaTriggerService, } from "./services.gen"; +import type { Retry } from "./types.gen"; import { OpenAPI } from "./core/OpenAPI"; import { isSuspendSignal, stepErrorMarker, taskErrorFromMarker } from "./wacError"; // import type { DenoS3LightClientSettings } from "./index"; @@ -420,12 +421,26 @@ export async function runScriptAsync( return _runScriptAsyncInternal(path, hash_, args, scheduledInSeconds, tag); } +/** Options for {@link runScriptByPathAsync} and {@link runScriptByHashAsync}, passed in + * place of `scheduledInSeconds`; a positional `tag` is then ignored. */ +export interface RunScriptAsyncOptions { + /** Schedule execution for a future time (in seconds) */ + scheduledInSeconds?: number; + /** Override the worker tag the job runs on */ + tag?: string; + /** Re-run the job when it fails, with the same policy a schedule takes, e.g. + * `{ constant: { attempts: 3, seconds: 10 } }`. The returned id is the first + * attempt's; each retry is a job of its own, recorded as a child of it. */ + retry?: Retry; +} + async function _runScriptAsyncInternal( path: string | null = null, hash_: string | null = null, args: Record | null = null, scheduledInSeconds: number | null = null, - tag: string | null = null + tag: string | null = null, + retry: Retry | null = null ): Promise { // Create a script job and return its job id. args = args || {}; @@ -439,6 +454,10 @@ async function _runScriptAsyncInternal( params["tag"] = tag; } + if (retry) { + params["retry"] = JSON.stringify(retry); + } + let parentJobId = getEnv("WM_JOB_ID"); if (parentJobId !== undefined) { params["parent_job"] = parentJobId; @@ -475,34 +494,47 @@ async function _runScriptAsyncInternal( * Run a script asynchronously by its path * @param path - Script path in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ export async function runScriptByPathAsync( path: string, args: Record | null = null, - scheduledInSeconds: number | null = null, + scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null ): Promise { - return _runScriptAsyncInternal(path, null, args, scheduledInSeconds, tag); + const o = asyncRunOptions(scheduledInSeconds, tag); + return _runScriptAsyncInternal(path, null, args, o.scheduledInSeconds, o.tag, o.retry); } /** * Run a script asynchronously by its hash * @param hash_ - Script hash in Windmill * @param args - Arguments to pass to the script - * @param scheduledInSeconds - Schedule execution for a future time (in seconds) + * @param scheduledInSeconds - Schedule execution for a future time (in seconds), or {@link RunScriptAsyncOptions} + * `{ scheduledInSeconds?, tag?, retry? }`, e.g. `{ retry: { constant: { attempts: 3, seconds: 10 } } }` * @param tag - Override the worker tag the job runs on * @returns Job ID of the created job */ export async function runScriptByHashAsync( hash_: string, args: Record | null = null, - scheduledInSeconds: number | null = null, + scheduledInSeconds: number | null | RunScriptAsyncOptions = null, tag: string | null = null ): Promise { - return _runScriptAsyncInternal(null, hash_, args, scheduledInSeconds, tag); + const o = asyncRunOptions(scheduledInSeconds, tag); + return _runScriptAsyncInternal(null, hash_, args, o.scheduledInSeconds, o.tag, o.retry); +} + +function asyncRunOptions( + scheduledInSeconds: number | null | RunScriptAsyncOptions, + tag: string | null +): RunScriptAsyncOptions { + return typeof scheduledInSeconds === "object" && scheduledInSeconds !== null + ? scheduledInSeconds + : { scheduledInSeconds: scheduledInSeconds ?? undefined, tag: tag ?? undefined }; } /** @@ -692,6 +724,19 @@ export async function getProgress(jobId?: any): Promise { }); } +/** + * Whether the worker running this job has received its shutdown signal. + * A draining worker waits for its current job to finish before exiting, so a + * long-running script can poll this and return early to let the worker be replaced. + * @param jobId? Job to check, defaults to the current job + */ +export async function workerIsDraining(jobId?: string): Promise { + return await JobService.workerIsDraining({ + id: jobId ?? getEnv("WM_JOB_ID") ?? "NO_JOB_ID", + workspace: getWorkspace(), + }); +} + /** * Set a flow user state * @param key key of the state @@ -1787,6 +1832,52 @@ export function setWorkflowCtx(ctx: WorkflowCtx | null) { Reflect.set(globalThis, "__wmill_wf_ctx", ctx); } +type TaskBodyScope = { run(store: true, fn: () => R): R; getStore(): true | undefined }; + +/** Marks the async context of a task body running in its child job. A flag on + * the ctx cannot stand in: the workflow body keeps replaying beside the task + * body (`Promise.all([a(), b()])`), and its calls must still park rather than + * run a sibling task's body in this job. Kept on `globalThis`, like the ctx, + * so a bundle holding two copies of this module shares one scope. Resolved + * through `process.getBuiltinModule` because a `node:` import would break + * browser bundles of this package. */ +function taskBodyScope(): TaskBodyScope | null { + const existing = Reflect.get(globalThis, "__wmill_wf_task_body"); + if (existing !== undefined) return existing; + let scope: TaskBodyScope | null = null; + try { + const AsyncLocalStorage = (globalThis as any).process?.getBuiltinModule?.("node:async_hooks")?.AsyncLocalStorage; + if (AsyncLocalStorage) scope = new AsyncLocalStorage(); + } catch {} + Reflect.set(globalThis, "__wmill_wf_task_body", scope); + return scope; +} + +function runTaskBody(fn: () => R): R { + const scope = taskBodyScope(); + return scope ? scope.run(true, fn) : fn(); +} + +function insideTaskBody(): boolean { + return taskBodyScope()?.getStore() === true; +} + +/** The ctx SDK calls dispatch through, or null where they run in place: outside + * a workflow, and inside a task body, which is one job and has no rounds of its + * own to dispatch or checkpoint a step in. */ +function dispatchCtx(): WorkflowCtx | null { + if (insideTaskBody()) return null; + return _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); +} + +function assertNotInsideTaskBody(what: string): void { + if (insideTaskBody()) { + throw new Error( + `${what} cannot be called from inside a task: a task runs as a single job and cannot suspend or dispatch steps. Call it from the workflow() body instead.`, + ); + } +} + export class WorkflowCtx { private completed: Record; @@ -2259,11 +2350,11 @@ export class WorkflowCtx { } export async function sleep(seconds: number): Promise { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + const ctx = dispatchCtx(); if (ctx) { return ctx._sleep(seconds) as Promise; } - // Outside workflow context, just wait locally + // Outside a workflow, or inside a task body, just wait locally await new Promise((r) => setTimeout(r, seconds * 1000)); } @@ -2274,12 +2365,15 @@ export async function sleep(seconds: number): Promise { * `fn`'s result is encoded as JSON and decoded back before it is returned, so * the round that runs the body sees the same types every replay sees: a `Date` * comes back as a string, a `Map` as `{}`. {@link Jsonified} is that shape. + * + * Called from inside a task body, `fn` runs in place and nothing is + * checkpointed: a task is a single job and is never replayed. */ export async function step( name: string, fn: () => T | Promise, ): Promise>> { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + const ctx = dispatchCtx(); if (ctx) { return ctx._runInlineStep(name, fn) as Promise>>; } @@ -2297,8 +2391,9 @@ export async function step( * const call_api = task(fetchOrders, { retry: { attempts: 3, delay: 30, multiplier: 2 } }); * * Inside a `workflow()`, calling a task dispatches it as a step. - * Outside a workflow, the function body executes directly and - * {@link TaskOptions} — retry included — does not apply. + * Outside a workflow, or called from inside another task, the function body + * executes directly and {@link TaskOptions} — retry included — does not apply: + * a task runs as a single job, so the tasks it calls run in that same job. * * A task runs as its own job, so its result is always encoded as JSON and * decoded back before the caller sees it: a `Date` comes back as a string, a @@ -2332,7 +2427,7 @@ export function task Promise>( // .then() which throws StepSuspend and empties pending before the caller // can flush. const wrapper = function (...args: any[]) { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + const ctx = dispatchCtx(); if (ctx) { // Inside a workflow with checkpoint/replay context — dispatch as step const script = taskPath ?? taskName; @@ -2352,7 +2447,7 @@ export function task Promise>( return (async () => { let result: any; try { - result = await fn(...args); + result = await runTaskBody(() => fn(...args)); } catch (e) { if (isSuspendSignal(e, StepSuspend)) throw e; ctx._raiseStepFailure(e); @@ -2361,7 +2456,7 @@ export function task Promise>( })(); } return stepResult; - } else if (getEnv("WM_JOB_ID") && !getEnv("WM_FLOW_JOB_ID")) { + } else if (!insideTaskBody() && getEnv("WM_JOB_ID") && !getEnv("WM_FLOW_JOB_ID")) { // Inside a Windmill root job without checkpoint context — v1 HTTP dispatch // WM_FLOW_JOB_ID is set on child jobs, so we skip dispatch for those return (async () => { @@ -2411,7 +2506,8 @@ export function taskScript(path: string, options?: TaskOptions): (...args: any[] assertUsableRetry(options?.retry); const name = path.split("/").pop() || path; const wrapper = function (...args: any[]) { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + assertNotInsideTaskBody(`taskScript("${path}")`); + const ctx = dispatchCtx(); if (ctx) { const kwargs = args.length === 1 && typeof args[0] === "object" && args[0] !== null ? args[0] @@ -2437,7 +2533,8 @@ export function taskFlow(path: string, options?: TaskOptions): (...args: any[]) assertUsableRetry(options?.retry); const name = path.split("/").pop() || path; const wrapper = function (...args: any[]) { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + assertNotInsideTaskBody(`taskFlow("${path}")`); + const ctx = dispatchCtx(); if (ctx) { const kwargs = args.length === 1 && typeof args[0] === "object" && args[0] !== null ? args[0] @@ -2490,7 +2587,8 @@ export function waitForApproval(options?: { skin?: "detailed" | "minimal"; description?: string | object; }): PromiseLike<{ value: any; approver: string; approved: boolean }> { - const ctx: WorkflowCtx | null = _workflowCtx ?? Reflect.get(globalThis, "__wmill_wf_ctx"); + assertNotInsideTaskBody("waitForApproval()"); + const ctx = dispatchCtx(); if (!ctx) { throw new Error("waitForApproval can only be called inside a workflow()"); } diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 488d265310..ffd7c2300a 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.826.0", + "version": "1.830.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts", "./wacError.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index a517f1ff0f..9adbfcec62 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.826.0", + "version": "1.830.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/typescript-client/tests/workflow_nested_task.test.ts b/typescript-client/tests/workflow_nested_task.test.ts new file mode 100644 index 0000000000..39ac2185d8 --- /dev/null +++ b/typescript-client/tests/workflow_nested_task.test.ts @@ -0,0 +1,91 @@ +/** + * A task body runs as one child job, so the SDK calls it makes cannot be + * dispatched as steps of their own. Against the real client. + * + * Run with: bun test typescript-client/tests/workflow_nested_task.test.ts + */ +import { expect, test, describe, mock, afterEach } from "bun:test"; + +mock.module("../services.gen", () => ({ + ResourceService: {}, + VariableService: {}, + JobService: {}, + HelpersService: {}, + AppService: {}, + MetricsService: {}, + OidcService: {}, + UserService: {}, + KafkaTriggerService: {}, +})); +mock.module("../core/OpenAPI", () => ({ + OpenAPI: { BASE: "http://localhost:8000/api", TOKEN: "tok" }, +})); + +const { WorkflowCtx, StepSuspend, task, taskScript, step, setWorkflowCtx } = await import( + "../client.ts" +); + +let ran: string[] = []; + +const inner = task(async function inner(x: string) { + ran.push("inner"); + return `inner:${x}`; +}); + +const outer = task(async function outer(x: string) { + ran.push("outer"); + await Promise.resolve(); + const viaTask = await inner(x); + const viaStep = await step("s", () => "stepped"); + return `${viaTask}+${viaStep}`; +}); + +const dispatching = task(async function dispatching() { + return await taskScript("f/other/script")({}); +}); + +/** Run `body` as the child round executing `key`, and hand back what it raised. */ +async function childRound(key: string, body: () => Promise): Promise { + setWorkflowCtx(new WorkflowCtx({ completed_steps: {}, _executing_key: key })); + const hung = new Promise((r) => setTimeout(() => r("hung"), 500)); + return await Promise.race([body().then(() => "returned", (e) => e), hung]); +} + +afterEach(() => { + ran = []; + setWorkflowCtx(null); +}); + +describe("SDK calls inside a task body", () => { + test("a nested task and step run in the task's own job", async () => { + const raised = await childRound("outer", async () => { + await outer("a"); + }); + + expect(raised).toBeInstanceOf(StepSuspend); + expect(raised.dispatchInfo).toMatchObject({ + mode: "step_complete", + result: "inner:a+stepped", + }); + }); + + // The workflow body keeps replaying beside the task body, and its own calls + // must stay parked: running them would execute a sibling task in this job. + test("a sibling task of the workflow body is not run", async () => { + const raised = await childRound("outer", async () => { + await Promise.all([outer("a"), inner("sibling")]); + }); + + expect(raised.dispatchInfo).toMatchObject({ mode: "step_complete" }); + expect(ran).toEqual(["outer", "inner"]); + }); + + test("a step that would need a dispatch fails instead of hanging", async () => { + const raised = await childRound("dispatching", async () => { + await dispatching(); + }); + + expect(raised).toBeInstanceOf(Error); + expect(raised.message).toContain("cannot be called from inside a task"); + }); +}); diff --git a/version.txt b/version.txt index 47ce201f9f..54f09db472 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.826.0 +1.830.0 diff --git a/windmill-yaml-validator/package-lock.json b/windmill-yaml-validator/package-lock.json index 0c496bc161..95c5128a13 100644 --- a/windmill-yaml-validator/package-lock.json +++ b/windmill-yaml-validator/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-yaml-validator", - "version": "1.826.0", + "version": "1.830.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-yaml-validator", - "version": "1.826.0", + "version": "1.830.0", "license": "Apache 2.0", "dependencies": { "@stoplight/yaml": "^4.3.0", diff --git a/windmill-yaml-validator/package.json b/windmill-yaml-validator/package.json index da525da3b8..6f2c2d7fed 100644 --- a/windmill-yaml-validator/package.json +++ b/windmill-yaml-validator/package.json @@ -1,6 +1,6 @@ { "name": "windmill-yaml-validator", - "version": "1.826.0", + "version": "1.830.0", "description": "YAML validator for Windmill flow, schedule, and trigger files", "main": "dist/index.js", "types": "dist/index.d.ts",